Security · python · 12 copies
Secrets & Credential Scanner
Scans source code repositories, configuration files, and directory trees for accidentally exposed secrets: API keys, passwords, tokens, private keys, AWS credentials, database connection strings, and JWT secrets. Checks git commit history (not just current files). Supports 50+ secret patterns including AWS, GCP, Azure, GitHub, Stripe, Slack, Twilio, SendGrid, and more. Essential for pre-commit CI/CD security gates and periodic audits.
- Linux
- Windows
- macOS
pip install "gitpython>=3.1" "pathspec>=0.11" "colorama>=0.4"#!/usr/bin/env python3
"""
Secrets & Credential Scanner
READ THE DOCUMENTATION BEFORE YOU RUN THIS:
https://permitany.com/scripts/secrets-scanner
It covers what the script does, whether it is safe to run, the
inventory file and credentials it needs, how to read the output,
and what to do when something fails.
pip install "gitpython>=3.1" "pathspec>=0.11" "colorama>=0.4"
"""
import argparse
import json
import math
import os
import re
import sys
from collections import defaultdict
from pathlib import Path
from typing import Optional
# Windows consoles default to a legacy code page (cp1252), which cannot encode
# the arrows, box-drawing and check-mark characters used in the output below.
# Printing one raises UnicodeEncodeError and kills the script mid-scan, so make
# stdout UTF-8 and degrade unencodable characters instead of crashing.
for _stream in (sys.stdout, sys.stderr):
try:
_stream.reconfigure(encoding="utf-8", errors="replace")
except (AttributeError, ValueError):
pass # already wrapped, or not a real stream (piped/redirected)
try:
import pathspec
except ImportError:
print("[!] Missing dependency: pip install pathspec")
sys.exit(1)
try:
from colorama import init, Fore, Style
init(autoreset=True)
except ImportError:
class Fore:
RED = YELLOW = GREEN = CYAN = WHITE = RESET = ""
class Style:
BRIGHT = RESET_ALL = ""
# ── Detection patterns organised by provider ─────────────────────────────────
_Q = "\"'" # both quote chars — used to build patterns without triple-quote conflicts
PATTERNS = {
# AWS
"aws_access_key_id": (r"AKIA[0-9A-Z]{16}", "CRITICAL"),
"aws_secret_access_key": (r"(?i)aws_secret_access_key\s*[=:]\s*['\"]?([A-Za-z0-9/+=]{40})['\"]?", "CRITICAL"),
"aws_session_token": (r"(?i)aws_session_token\s*[=:]\s*['\"]?([A-Za-z0-9/+=]{100,})['\"]?", "HIGH"),
# GCP
"gcp_api_key": (r"AIza[0-9A-Za-z\-_]{35}", "CRITICAL"),
"gcp_service_account": (r'"type"\s*:\s*"service_account"', "HIGH"),
# Azure
"azure_connection_str": (r"(?i)DefaultEndpointsProtocol=https;AccountName=[^;]+;AccountKey=[A-Za-z0-9+/=]{80,}", "CRITICAL"),
"azure_client_secret": (r"(?i)client_secret\s*[=:]\s*['\"]?[a-zA-Z0-9_\-~]{34,}['\"]?", "HIGH"),
# GitHub
"github_pat_classic": (r"ghp_[0-9a-zA-Z]{36}", "CRITICAL"),
"github_pat_fine": (r"github_pat_[0-9a-zA-Z]{22}_[0-9a-zA-Z]{59}", "CRITICAL"),
"github_oauth": (r"gho_[0-9a-zA-Z]{36}", "CRITICAL"),
"github_server_to_server": (r"ghs_[0-9a-zA-Z]{36}", "HIGH"),
# Private Keys
"rsa_private_key": (r"-----BEGIN RSA PRIVATE KEY-----", "CRITICAL"),
"ec_private_key": (r"-----BEGIN EC PRIVATE KEY-----", "CRITICAL"),
"openssh_private_key": (r"-----BEGIN OPENSSH PRIVATE KEY-----", "CRITICAL"),
"pkcs8_private_key": (r"-----BEGIN PRIVATE KEY-----", "CRITICAL"),
"pgp_private_key": (r"-----BEGIN PGP PRIVATE KEY BLOCK-----", "HIGH"),
# Passwords in code (using character class to avoid nested quote issues)
"password_assignment": (r"(?i)password\s*[=:]\s*[\"'][^\"']{8,}[\"']", "HIGH"),
"passwd_assignment": (r"(?i)passwd\s*[=:]\s*[\"'][^\"']{8,}[\"']", "HIGH"),
"pwd_assignment": (r"(?i)\bpwd\s*=\s*[\"'][^\"']{8,}[\"']", "MEDIUM"),
"secret_assignment": (r"(?i)\bsecret\s*[=:]\s*[\"'][^\"']{12,}[\"']", "HIGH"),
# Database URLs
"postgresql_url": (r"postgresql://[^:]+:[^@]+@[^/\s]+", "CRITICAL"),
"mysql_url": (r"mysql://[^:]+:[^@]+@[^/\s]+", "CRITICAL"),
"mongodb_url": (r"mongodb(?:\+srv)?://[^:]+:[^@]+@[^/\s]+", "CRITICAL"),
"redis_url": (r"redis://:[^@]+@[^/\s]+", "HIGH"),
"mssql_conn": (r"(?i)Server=[^;]+;.*Password=[^;]+", "HIGH"),
# JWT
"jwt_secret": (r"(?i)jwt[_\-]?secret\s*[=:]\s*[\"'][^\"']{12,}[\"']", "HIGH"),
"jwt_signing_key": (r"(?i)signing[_\-]?key\s*[=:]\s*[\"'][^\"']{12,}[\"']", "HIGH"),
# Generic API Keys
"generic_api_key": (r"(?i)api[_\-]?key\s*[=:]\s*[\"'][a-zA-Z0-9\-_]{20,}[\"']", "HIGH"),
"generic_token": (r"(?i)\btoken\s*[=:]\s*[\"'][a-zA-Z0-9\-_.]{20,}[\"']", "MEDIUM"),
"generic_auth_token": (r"(?i)auth[_\-]?token\s*[=:]\s*[\"'][a-zA-Z0-9\-_.]{20,}[\"']", "HIGH"),
# Stripe
"stripe_secret_key": (r"sk_live_[0-9a-zA-Z]{24}", "CRITICAL"),
"stripe_public_key": (r"pk_live_[0-9a-zA-Z]{24}", "MEDIUM"),
"stripe_restricted": (r"rk_live_[0-9a-zA-Z]{24}", "HIGH"),
# Slack
"slack_bot_token": (r"xoxb-[0-9]{11}-[0-9]{11}-[0-9a-zA-Z]{24}", "CRITICAL"),
"slack_user_token": (r"xoxp-[0-9]{11}-[0-9]{11}-[0-9]{11}-[0-9a-f]{32}", "CRITICAL"),
"slack_webhook": (r"https://hooks\.slack\.com/services/T[A-Z0-9]{8}/B[A-Z0-9]{8}/[A-Za-z0-9]{24}", "HIGH"),
# Twilio
"twilio_account_sid": (r"AC[a-z0-9]{32}", "HIGH"),
"twilio_auth_token": (r"(?i)twilio.*[\"'][0-9a-f]{32}[\"']", "HIGH"),
# SendGrid
"sendgrid_api_key": (r"SG\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9_-]{43}", "CRITICAL"),
# Others
"mailgun_api_key": (r"key-[0-9a-zA-Z]{32}", "HIGH"),
"npm_token": (r"//registry\.npmjs\.org/:_authToken=\S+", "CRITICAL"),
"pypi_token": (r"pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,}", "HIGH"),
"docker_registry_auth": (r'"auth"\s*:\s*"[A-Za-z0-9+/=]{16,}"', "HIGH"),
"basic_auth_url": (r"https?://[^:]+:[^@]{8,}@", "HIGH"),
"google_oauth_client": (r"[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com", "MEDIUM"),
"firebase_api_key": (r'"apiKey"\s*:\s*"AIza[0-9A-Za-z\-_]{35}"', "HIGH"),
"vault_token": (r"hvs\.[A-Za-z0-9]{24,}", "CRITICAL"),
"datadog_api_key": (r"(?i)datadog.*[\"'][0-9a-f]{32}[\"']", "HIGH"),
"pagerduty_api_key": (r"(?i)pagerduty.*[\"'][a-zA-Z0-9\-_+]{20,}[\"']", "HIGH"),
"telegram_bot_token": (r"[0-9]{8,10}:[a-zA-Z0-9_-]{35}", "HIGH"),
"shopify_token": (r"shpat_[a-fA-F0-9]{32}", "CRITICAL"),
"linear_api_key": (r"lin_api_[a-zA-Z0-9]{40}", "HIGH"),
"heroku_api_key": (r"(?i)heroku.*[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", "MEDIUM"),
}
BINARY_EXTENSIONS = {
".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico",
".pdf", ".zip", ".tar", ".gz", ".7z", ".rar",
".exe", ".dll", ".so", ".dylib", ".bin",
".pyc", ".pyo", ".class",
".mp3", ".mp4", ".mov", ".avi",
".ttf", ".woff", ".woff2", ".eot",
".lock",
}
DEFAULT_EXCLUDES = [
"node_modules", ".venv", "venv", "dist", "build", ".git",
"__pycache__", ".idea", ".vscode", "vendor", "coverage",
]
def calculate_entropy(s: str) -> float:
"""Shannon entropy of a string — high values suggest random tokens/keys."""
if not s:
return 0.0
freq = defaultdict(int)
for c in s:
freq[c] += 1
n = len(s)
return -sum((count / n) * math.log2(count / n) for count in freq.values())
def redact_secret(value: str) -> str:
"""Show first 4 and last 4 characters only: AKIA****MPLE."""
if len(value) <= 10:
return "****"
return value[:4] + "****" + value[-4:]
def classify_severity(pattern_name: str, match: str) -> str:
_, sev = PATTERNS.get(pattern_name, ("", "MEDIUM"))
if "test" in match.lower() or "example" in match.lower() or "sample" in match.lower():
sev = "LOW" if sev in ("MEDIUM", "LOW") else sev
return sev
def scan_file(filepath: Path) -> list:
"""Scan a single file for secret patterns; return list of findings."""
findings = []
try:
text = filepath.read_text(encoding="utf-8", errors="replace")
for line_no, line in enumerate(text.splitlines(), 1):
for name, (regex, _) in PATTERNS.items():
for m in re.finditer(regex, line):
val = m.group(0)
findings.append({
"file": str(filepath),
"line": line_no,
"pattern": name,
"severity": classify_severity(name, val),
"redacted": redact_secret(val),
"entropy": round(calculate_entropy(val), 2),
"source": "filesystem",
})
except Exception:
pass
return findings
def scan_git_history(repo_path: str, depth: int) -> list:
"""Scan git commit diffs for secrets — finds secrets in deleted files too."""
findings = []
try:
import git
repo = git.Repo(repo_path)
commits = list(repo.iter_commits("HEAD", max_count=depth))
for commit in commits:
try:
if not commit.parents:
continue
diffs = commit.parents[0].diff(commit, create_patch=True)
for diff in diffs:
try:
patch = diff.diff.decode("utf-8", errors="replace") if diff.diff else ""
except Exception:
continue
for line_no, line in enumerate(patch.splitlines(), 1):
if not line.startswith("+"):
continue
for name, (regex, _) in PATTERNS.items():
for m in re.finditer(regex, line):
val = m.group(0)
findings.append({
"file": diff.b_path or diff.a_path,
"line": line_no,
"pattern": name,
"severity": classify_severity(name, val),
"redacted": redact_secret(val),
"commit": commit.hexsha[:7],
"commit_date": commit.committed_datetime.isoformat(),
"author": str(commit.author),
"source": "git_history",
})
except Exception:
continue
except ImportError:
print(f"{Fore.YELLOW}[!] gitpython not installed — skipping git history scan")
except Exception as exc:
print(f"{Fore.YELLOW}[!] Git history scan error: {exc}")
return findings
def scan_directory(path: str, exclude_patterns: list, entropy_threshold: float):
"""Walk directory and scan every non-binary, non-excluded file."""
all_findings = []
spec = pathspec.PathSpec.from_lines("gitwildmatch", exclude_patterns)
root = Path(path)
scanned = 0
for fp in root.rglob("*"):
if not fp.is_file():
continue
try:
rel = str(fp.relative_to(root))
except ValueError:
continue
if spec.match_file(rel):
continue
if fp.suffix.lower() in BINARY_EXTENSIONS:
continue
if fp.stat().st_size > 5_000_000:
continue
all_findings.extend(scan_file(fp))
scanned += 1
# High-entropy string sweep
try:
text = fp.read_text(encoding="utf-8", errors="replace")
for line_no, line in enumerate(text.splitlines(), 1):
for token in re.findall(r"[\"']([A-Za-z0-9+/=_\-]{20,64})[\"']", line):
ent = calculate_entropy(token)
if ent >= entropy_threshold:
caught = any(re.search(v[0], token) for _, v in PATTERNS.items())
if not caught:
all_findings.append({
"file": str(fp),
"line": line_no,
"pattern": "high_entropy_string",
"severity": "MEDIUM",
"redacted": redact_secret(token),
"entropy": round(ent, 2),
"source": "entropy",
})
except Exception:
pass
return all_findings, scanned
def print_finding(f: dict):
sev = f["severity"]
color = Fore.RED if sev == "CRITICAL" else Fore.YELLOW if sev == "HIGH" else Fore.CYAN
src_tag = " (git history)" if f.get("source") == "git_history" else ""
print(f"\n {color}x {sev} {f['file']}:{f['line']}{src_tag}")
print(f" Pattern: {f['pattern']}")
print(f" Found: {f['redacted']} (redacted)")
if f.get("commit"):
print(f" Commit: {f['commit']} ({f.get('commit_date', '')})")
print(f" {Fore.WHITE} -> Key was deleted but visible in git history. Rotate immediately.")
print(f" Fix: BFG Repo Cleaner to purge history, then force-push")
def main():
ap = argparse.ArgumentParser(description="Secrets & Credential Scanner")
ap.add_argument("--path", required=True, help="Directory or git repo root to scan")
ap.add_argument("--history", type=int, default=50, help="Git history depth (default 50)")
ap.add_argument("--output", default="secrets_report.json")
ap.add_argument("--exclude", default="", help="Comma-separated extra exclude patterns")
ap.add_argument("--entropy", type=float, default=4.5,
help="Min Shannon entropy for high-entropy detection (default 4.5)")
args = ap.parse_args()
excludes = DEFAULT_EXCLUDES[:]
if args.exclude:
excludes.extend([p.strip() for p in args.exclude.split(",") if p.strip()])
print(f"{Fore.CYAN}[*] Secrets Scanner | Path: {args.path} | Git history: {args.history} commits")
print(f"{Fore.CYAN}[*] Loading {len(PATTERNS)} detection patterns + entropy analysis (threshold: {args.entropy})")
print(f"{Fore.CYAN}[*] Excluding: {', '.join(excludes)}")
fs_findings, scanned = scan_directory(args.path, excludes, args.entropy)
print(f"{Fore.CYAN}[*] Scanned {scanned} files")
hist_findings = scan_git_history(args.path, args.history)
all_findings = fs_findings + hist_findings
live = [f for f in all_findings if f.get("source") == "filesystem"]
history_hits = [f for f in all_findings if f.get("source") == "git_history"]
entropy_hits = [f for f in all_findings if f.get("source") == "entropy"]
if live:
print(f"\n{Fore.RED}[!] LIVE SECRETS FOUND")
for f in sorted(live, key=lambda x: ["CRITICAL", "HIGH", "MEDIUM", "LOW"].index(x["severity"])):
print_finding(f)
else:
print(f"\n{Fore.GREEN}[+] No live secrets found in current files")
if history_hits:
print(f"\n{Fore.YELLOW}[!] SECRETS IN GIT HISTORY (deleted but still accessible)")
for f in history_hits:
print_finding(f)
if entropy_hits:
print(f"\n{Fore.CYAN}[*] HIGH ENTROPY STRINGS (possible tokens — manual review needed)")
for f in entropy_hits[:5]:
print(f" {Fore.CYAN}[?] MEDIUM {f['file']}:{f['line']} entropy={f['entropy']}")
print(f" Value: {f['redacted']} — matches random token pattern")
c_counts = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0}
for f in all_findings:
c_counts[f.get("severity", "LOW")] += 1
print(f"\n{Fore.WHITE}{Style.BRIGHT}Summary: "
f"{Fore.RED}{c_counts['CRITICAL']} CRITICAL{Fore.WHITE} | "
f"{Fore.YELLOW}{c_counts['HIGH']} HIGH{Fore.WHITE} | "
f"{Fore.CYAN}{c_counts['MEDIUM']} MEDIUM{Fore.WHITE} | "
f"{Fore.GREEN}{c_counts['LOW']} LOW")
if c_counts["CRITICAL"] > 0:
print(f"{Fore.RED}ACTION REQUIRED: Rotate all CRITICAL secrets immediately.")
report = {
"scan_path": args.path,
"patterns_checked": len(PATTERNS),
"files_scanned": scanned,
"git_commits_checked": args.history,
"findings": all_findings,
"summary": c_counts,
}
with open(args.output, "w") as fh:
json.dump(report, fh, indent=2, default=str)
print(f"{Fore.CYAN}[*] Report saved: {args.output}")
return 2 if c_counts['CRITICAL'] else (1 if c_counts['HIGH'] or c_counts['MEDIUM'] else 0)
if __name__ == "__main__":
try:
sys.exit(main())
except KeyboardInterrupt:
print("\nInterrupted.", file=sys.stderr)
sys.exit(130)
--pathtextrequiredDirectory or git repository root to scan.
--historynumberHow many commits of git history to scan. Default 50. This is the flag that finds the secret that was committed and later removed.
--outputtextJSON report path. Default: secrets_report.json.
--excludetextComma-separated extra exclude patterns, on top of the defaults.
--entropynumberShannon entropy threshold for flagging a high-entropy string. Default 4.5. Lower finds more and reports more false positives.
What This Solves
Secrets get committed. Everyone knows not to, and it happens anyway - a quick test with a real key, a config file that was supposed to be in .gitignore, a `.env` copied into the repo to get something working on a Friday. The problem is not finding it in the current files. The problem is that **removing it in a later commit does not remove it.** The secret is still in the history, and anyone who can clone the repository can read it. Most "we removed that key" conversations are about a key that is still there. This scans a directory tree or a git repository for: API keys, tokens, passwords private keys AWS credentials database connection strings JWT secrets high-entropy strings that look like credentials and - with `--history` - **the commit history as well as the current files**, which is the part that matters. Run it before a repository goes public, as a pre-commit hook, and once over every repository you inherited.
Is It Safe to Run
Yes. It reads files on your own machine and reports. It never contacts a network, never uploads anything, never modifies the repository, and never rewrites history. Two things to be careful about, both about the **output** rather than the scan: The report contains the secrets it found. That is unavoidable - a finding you cannot see is not a finding - but it means `secrets_report.json` is itself a file full of credentials. Do not commit it. Do not paste it into a ticket. Treat it exactly as you would treat the secrets in it. Finding a secret means it is already compromised. If it was pushed anywhere - a shared remote, a fork, a CI system, a laptop that cloned the repo - assume it is known. **Rotate the credential first, then clean the history.** Doing it in the other order means spending an afternoon rewriting git while the live key is still valid.
Before You Start
1. Python 3.9 or newer: python --version 2. The libraries: pip install "gitpython>=3.1" "pathspec>=0.11" "colorama>=0.4" 3. For --history: the path must be a git repository with its history present. A shallow clone has nothing to scan - CI checkouts are shallow by default, which is the usual reason a pipeline scan finds nothing.
Your First Run
Point it at a repository you own: python secrets_scanner.py --path ~/code/myapp That scans the current files. Read the findings and discount the false positives - every secrets scanner produces some, and knowing what yours look like comes first. Then the part that matters: python secrets_scanner.py --path ~/code/myapp --history 200 This walks the last 200 commits. **Expect it to find things the current-files scan did not**, because that is the entire point: a secret that was committed and later deleted is invisible to the first scan and fully readable to anyone with a clone.
What the Output Means
Findings are graded CRITICAL, HIGH, MEDIUM or LOW, with the file, the line, and the matched content. Reading it: A pattern match vs an entropy match A match on a known shape - an AWS key id, a private key header, a recognisable token prefix - is high confidence. A **high-entropy string** is a guess: the content looks random enough to be a credential. Entropy findings are where almost all the false positives live. A minified bundle, a base64 blob, a test fixture and a UUID all look like secrets to an entropy check. --entropy 4.5 is the dial Lower it and you find more real secrets and far more noise. Raise it and you miss short credentials. 4.5 is a reasonable default; tune it per repository rather than globally. A finding in history names the commit Which tells you when it was introduced and by whom, and whether the branch it is on ever left the machine. CRITICAL findings Private keys and live-looking cloud credentials. These are the ones to act on before you finish reading the report. What the script cannot tell you, and you must decide: Whether the secret is still valid. It finds strings; it does not test them. A key from 2019 that was rotated twice is a cleanup task, not an incident. A key from last month probably is not. Whether the repository is private. A secret in a private repository with four people on it and one in a repository that was public for a week are different problems with the same finding.
Exit Codes for Cron and CI
0 nothing above low severity 1 high or medium findings 2 critical findings 130 you pressed Ctrl-C As a CI gate, failing only on critical: python secrets_scanner.py --path . --output report.json test $? -lt 2 **In CI, remember the shallow clone.** A default checkout has depth 1, so `--history` sees one commit. Fetch the full history if you want the history scanned, or accept that CI only checks the current files and run the deep scan separately. As a pre-commit hook, the current-files scan on the working tree is fast and catches the thing before it becomes a history problem at all - which is by far the cheapest place to catch it.
If It Finds Something Real
The order matters, and most people get it backwards. 1. **Rotate the credential.** Immediately, before anything else. The secret is in a repository; assume it is known. 2. Work out the exposure. Was the repository ever public? Who has clones? Did it pass through CI, where it may be in build logs and caches too? 3. Then clean the history - `git filter-repo` or BFG. This rewrites commits, so every clone must be re-cloned and every open branch rebased. It is disruptive, which is why it comes after the rotation rather than instead of it. 4. Add the pattern to a pre-commit hook so the next one is caught before it is committed. Cleaning history without rotating is the common mistake. It is a lot of work that leaves the live credential live.
Every Flag
--path PATH REQUIRED. Directory or git repository root. --history N Scan this many commits of git history. Default 50. **This is the flag that finds the secret that was committed and later removed.** Needs a full clone. --output PATH JSON report. Default secrets_report.json. The report contains the secrets - treat it accordingly. --exclude LIST Comma-separated extra exclude patterns, on top of the defaults. --entropy F Shannon entropy threshold for flagging a high-entropy string. Default 4.5. Lower finds more and reports far more false positives.
When Something Fails
ModuleNotFoundError `pip install "gitpython>=3.1" "pathspec>=0.11" "colorama>=0.4"`. --history finds nothing on a repository you know has history A shallow clone. `git log --oneline | wc -l` - if it says 1, there is nothing to scan. Fetch the full history first. This is the default state of a CI checkout. Hundreds of findings, almost all noise Entropy matches on build artifacts. Exclude the directories: `--exclude node_modules,dist,build,vendor,.min.js`, and raise --entropy for that repository. A real secret the scan missed Short credentials fall below the entropy threshold, and an unusual format has no pattern to match. Lower --entropy for a targeted re-scan, and grep for the specific shape if you know it. A finding in a file that is in .gitignore The file is on disk and was scanned. If it was never committed, it is not in the repository - check with `git log --all -- <path>`. Still worth fixing the local copy. The scan is very slow --history on a large repository reads every blob in every scanned commit. Lower the depth, or scan history once rather than on every run and keep the fast current-files scan in the hook. You want to know whether a found key is still live This script will not test it - testing a credential is an action with consequences. Check it in the provider's console, or rotate it and find out that way.