ci: scan pull requests for credentials and injection with ThreatCrush - #22293

#!/bin/bash
set -eu
file=".github/workflows/threatcrush-scan.yml"
wc -l "$file"
sed -n '250,315p' "$file"
rg -n "listComments|paginate|createComment|ThreatCrush Security Scan|github-actions\\[bot\\]|per_page" "$file"

#!/bin/bash
set -euo pipefail

spec='`@profullstack/threatcrush`@0.11.0'
tmpdir="$(mktemp -d)"
trap 'rm -rf "$tmpdir"' EXIT

name="$(npm pack --ignore-scripts --silent --pack-destination "$tmpdir" "$spec" | tail -n1)"
tarball="$tmpdir/$name"

echo '--- lock or shrinkwrap files in the reviewed tarball ---'
tar -tf "$tarball" | grep -E '^package/(npm-shrinkwrap\.json|package-lock\.json)$' || true

echo '--- package dependency declarations ---'
tar -xOf "$tarball" package/package.json |
  jq '{name, version, dependencies, bundledDependencies, bundleDependencies}'

echo '--- registry dependency metadata ---'
npm view "$spec" dist.integrity dependencies bundledDependencies --json

#!/bin/bash
set -eu
printf '%s\n' '--- relevant repository references ---'
rg -n -C 4 'threatcrush|Unrecognised|footer|finding|expected' .github/scripts .github/workflows . 2>/dev/null | head -n 240
printf '%s\n' '--- tracked candidate files ---'
git ls-files | rg '(^|/)(threatcrush|README|package|pyproject|workflow)' | head -n 160

#!/bin/bash
set -eu
printf '%s\n' '--- workflow scan and conversion steps ---'
sed -n '55,215p' .github/workflows/threatcrush-scan.yml
printf '%s\n' '--- parser definitions and parse loop ---'
sed -n '28,118p' .github/scripts/threatcrush-to-sarif.py
printf '%s\n' '--- package metadata ---'
curl -fsSL https://registry.npmjs.org/@profullstack%2fthreatcrush | python3 - <<'PY'
import json, sys
data = json.load(sys.stdin)
print("dist-tags:", data.get("dist-tags"))
print("versions:", sorted(data.get("versions", {}), key=lambda x: tuple(int(p) if p.isdigit() else p for p in x.split("."))))
for version, meta in data.get("versions", {}).items():
    print(version, meta.get("dist", {}).get("tarball"), meta.get("dist", {}).get("integrity"))
PY

#!/bin/bash
set -eu
tmp=$(mktemp)
trap 'rm -f "$tmp" "$tmp.tgz"' EXIT

curl -fsSL -o "$tmp" 'https://registry.npmjs.org/@profullstack%2fthreatcrush'
python3 - "$tmp" <<'PY'
import json, sys
data = json.load(open(sys.argv[1]))
print("dist-tags:", data.get("dist-tags"))
versions = data.get("versions", {})
print("published versions:", ", ".join(sorted(versions)))
for version in sorted(versions):
    meta = versions[version]
    print(version, meta.get("dist", {}).get("tarball"), meta.get("dist", {}).get("integrity"))
PY

tarball=$(python3 - "$tmp" <<'PY'
import json, sys
data = json.load(open(sys.argv[1]))
print(data["versions"]["0.11.0"]["dist"]["tarball"])
PY
)
curl -fsSL -o "$tmp.tgz" "$tarball"
printf '%s\n' '--- 0.11.0 package file list ---'
tar -tzf "$tmp.tgz" | head -n 120
printf '%s\n' '--- likely formatter/footer strings ---'
tar -xOzf "$tmp.tgz" 2>/dev/null \
  | strings \
  | rg -n -i 'issue|security issue|found|scan|finding|format' \
  | head -n 200 || true

#!/bin/bash
set -eu
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT

for version in 0.1.0 0.1.16 0.2.0 0.2.1 0.2.2 0.3.0 0.4.0 0.5.1 0.6.0 0.6.1 0.6.2 0.7.0 0.7.1 0.7.2 0.8.0 0.9.0 0.10.0 0.11.0; do
  meta="$tmpdir/$version.json"
  curl -fsSL -o "$meta" "https://registry.npmjs.org/@profullstack%2fthreatcrush/$version"
  tarball=$(python3 - "$meta" <<'PY'
import json, sys
print(json.load(open(sys.argv[1]))["dist"]["tarball"])
PY
)
  archive="$tmpdir/$version.tgz"
  curl -fsSL -o "$archive" "$tarball"
  printf '\n=== %s package.json ===\n' "$version"
  tar -xOzf "$archive" package/package.json
  printf '\n=== %s footer/count-related source ===\n' "$version"
  for file in $(tar -tzf "$archive" | rg '^package/(dist/.*\.(js|map)|README\.md)$'); do
    content=$(tar -xOzf "$archive" "$file" 2>/dev/null || true)
    matches=$(printf '%s' "$content" | rg -n -i -C 3 'issue|security issue|finding|found|total|count|format' || true)
    if [ -n "$matches" ]; then
      printf '%s\n' "--- $file ---"
      printf '%s\n' "$matches" | head -n 100
    fi
  done
done

#!/bin/bash
set -eu
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT

for version in 0.1.0 0.1.16 0.2.0 0.2.1 0.2.2 0.3.0 0.4.0 0.5.1 0.6.0 0.6.1 0.6.2 0.7.0 0.7.1 0.7.2 0.8.0 0.9.0 0.10.0 0.11.0; do
  meta="$tmpdir/$version.json"
  curl -fsSL -o "$meta" "https://registry.npmjs.org/@profullstack%2fthreatcrush/$version"
  tarball=$(python3 - "$meta" <<'PY'
import json, sys
print(json.load(open(sys.argv[1]))["dist"]["tarball"])
PY
)
  archive="$tmpdir/$version.tgz"
  curl -fsSL -o "$archive" "$tarball"

  python3 - "$version" "$archive" <<'PY'
import json, re, sys, tarfile

version, archive = sys.argv[1:]
terms = re.compile(r'issue|security issue|finding|found|total|count|format', re.I)
literal = re.compile(r"""(['"`])((?:\\.|(?!\1).)*)\1""")

with tarfile.open(archive, "r:gz") as tf:
    print(f"\n=== {version} ===")
    pkg = json.load(tf.extractfile("package/package.json"))
    print("bin:", pkg.get("bin"), "main:", pkg.get("main"), "version:", pkg.get("version"))

    members = {m.name: m for m in tf.getmembers()}
    for name, member in members.items():
        if name.endswith(".map"):
            try:
                sm = json.load(tf.extractfile(member))
            except Exception:
                continue
            for source_name, source in zip(sm.get("sources", []), sm.get("sourcesContent", [])):
                if not source:
                    continue
                lines = source.splitlines()
                hits = [i for i, line in enumerate(lines) if terms.search(line)]
                if hits:
                    print(f"-- source {source_name} --")
                    emitted = set()
                    for i in hits:
                        for j in range(max(0, i-1), min(len(lines), i+2)):
                            if j not in emitted:
                                print(f"{j+1}: {lines[j][:300]}")
                                emitted.add(j)
                            if len(emitted) >= 45:
                                break
                        if len(emitted) >= 45:
                            break

        elif name.endswith(".js"):
            text = tf.extractfile(member).read().decode("utf-8", "replace")
            vals = []
            for match in literal.finditer(text):
                value = match.group(2)
                if terms.search(value):
                    value = value.replace("\\n", "\\n").replace("\\r", "\\r")
                    if value not in vals:
                        vals.append(value)
            if vals:
                print(f"-- literals {name} --")
                for value in vals[:80]:
                    print(repr(value[:300]))
PY
done

#!/bin/bash
set -eu
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT

for version in 0.1.0 0.2.2 0.3.0 0.10.0 0.11.0; do
  meta="$tmpdir/$version.json"
  curl -fsSL -o "$meta" "https://registry.npmjs.org/@profullstack%2fthreatcrush/$version"
  tarball=$(python3 - "$meta" <<'PY'
import json, sys
print(json.load(open(sys.argv[1]))["dist"]["tarball"])
PY
)
  archive="$tmpdir/$version.tgz"
  curl -fsSL -o "$archive" "$tarball"

  python3 - "$version" "$archive" <<'PY'
import json, sys, tarfile

version, archive = sys.argv[1:]
with tarfile.open(archive, "r:gz") as tf:
    print(f"\n=== {version} ===")
    candidates = []
    for member in tf.getmembers():
        if not member.name.endswith(".map"):
            continue
        try:
            sm = json.load(tf.extractfile(member))
        except Exception:
            continue
        for name, source in zip(sm.get("sources", []), sm.get("sourcesContent", [])):
            if source and ("commands/scan" in name or name.endswith("/scan.ts") or name.endswith("/scan.js")):
                candidates.append((name, source))

    if not candidates:
        print("No scan source map found.")
    for name, source in candidates[:5]:
        print(f"-- {name} --")
        lines = source.splitlines()
        for i, line in enumerate(lines):
            if any(term in line for term in (
                "issue(s) found", "No security issues found", "findings.length",
                "console.log", "format", "summary", "findings",
            )):
                start, end = max(0, i - 2), min(len(lines), i + 3)
                print(f"{start+1}-{end}:")
                for j in range(start, end):
                    print(f"{j+1}: {lines[j][:240]}")
PY
done

#!/bin/bash
set -eu
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT

for version in 0.1.0 0.1.1 0.1.2 0.1.3 0.1.4 0.1.5 0.1.6 0.1.7 0.1.8 0.1.9 0.1.10 0.1.11 0.1.12 0.1.13 0.1.14 0.1.15 0.1.16 0.2.0 0.2.1 0.2.2; do
  meta="$tmpdir/$version.json"
  curl -fsSL -o "$meta" "https://registry.npmjs.org/@profullstack%2fthreatcrush/$version"
  tarball=$(python3 - "$meta" <<'PY'
import json, sys
print(json.load(open(sys.argv[1]))["dist"]["tarball"])
PY
)
  archive="$tmpdir/$version.tgz"
  curl -fsSL -o "$archive" "$tarball"

  python3 - "$version" "$archive" <<'PY'
import json, re, sys, tarfile

version, archive = sys.argv[1:]
terms = re.compile(r'issue|security issue|finding|found|format', re.I)

with tarfile.open(archive, "r:gz") as tf:
    pkg = json.load(tf.extractfile("package/package.json"))
    print(f"\n=== {version} ===")
    print("bin:", pkg.get("bin"), "main:", pkg.get("main"))
    for member in tf.getmembers():
        if not member.name.endswith((".js", ".mjs", ".cjs")):
            continue
        text = tf.extractfile(member).read().decode("utf-8", "replace")
        hits = [m.start() for m in terms.finditer(text)]
        if not hits:
            continue
        print(f"-- {member.name}: {len(hits)} matches --")
        emitted = set()
        for pos in hits:
            start = max(0, pos - 180)
            end = min(len(text), pos + 260)
            snippet = re.sub(r"\s+", " ", text[start:end])
            if snippet not in emitted:
                print(snippet[:500])
                emitted.add(snippet)
            if len(emitted) >= 12:
                break
PY
done

#!/bin/bash
set -eu
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT

for version in 0.1.0 0.1.16 0.2.0 0.2.1 0.2.2; do
  meta="$tmpdir/$version.json"
  curl -fsSL -o "$meta" "https://registry.npmjs.org/@profullstack%2fthreatcrush/$version"
  tarball=$(python3 - "$meta" <<'PY'
import json, sys
print(json.load(open(sys.argv[1]))["dist"]["tarball"])
PY
)
  archive="$tmpdir/$version.tgz"
  curl -fsSL -o "$archive" "$tarball"

  python3 - "$version" "$archive" <<'PY'
import json, re, sys, tarfile

version, archive = sys.argv[1:]
string_re = re.compile(r"""(['"`])((?:\\.|(?!\1).)*)\1""")
terms = re.compile(r'scan|issue|finding|security|vulnerab|file:|info:|found|result', re.I)

with tarfile.open(archive, "r:gz") as tf:
    print(f"\n=== {version} ===")
    pkg = json.load(tf.extractfile("package/package.json"))
    print(json.dumps({
        k: pkg.get(k) for k in ("name", "version", "description", "bin", "main", "scripts", "dependencies")
    }, indent=2))
    for name in ("package/README.md", "package/README", "package/dist/index.js"):
        try:
            text = tf.extractfile(name).read().decode("utf-8", "replace")
        except KeyError:
            continue
        if name.endswith("README.md") or name.endswith("README"):
            print(f"-- {name} --")
            print(text[:6000])
        else:
            values = []
            for match in string_re.finditer(text):
                value = match.group(2)
                if terms.search(value) and value not in values:
                    values.append(value)
            print(f"-- matching literals {name} --")
            for value in values[:160]:
                print(repr(value[:500]))
PY
done

#!/bin/bash
set -eu
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT

python3 - "$tmpdir" <<'PY'
import json, os, subprocess, sys, tarfile

tmpdir = sys.argv[1]
versions = [
    "0.1.0", "0.1.1", "0.1.16",
    "0.2.0", "0.2.1", "0.2.2",
    "0.3.0", "0.4.0", "0.5.1", "0.6.0", "0.7.0",
    "0.8.0", "0.9.0", "0.10.0", "0.11.0",
]

for version in versions:
    meta_path = os.path.join(tmpdir, version + ".json")
    archive_path = os.path.join(tmpdir, version + ".tgz")
    subprocess.run([
        "curl", "-fsSL", "-o", meta_path,
        f"https://registry.npmjs.org/@profullstack%2fthreatcrush/{version}",
    ], check=True, stdout=subprocess.DEVNULL)
    meta = json.load(open(meta_path))
    subprocess.run([
        "curl", "-fsSL", "-o", archive_path,
        meta["dist"]["tarball"],
    ], check=True, stdout=subprocess.DEVNULL)

    text = []
    with tarfile.open(archive_path, "r:gz") as tf:
        for member in tf.getmembers():
            if member.isfile() and member.name.endswith((".js", ".map", ".md")):
                try:
                    text.append(tf.extractfile(member).read().decode("utf-8", "replace"))
                except Exception:
                    pass
    bundle = "\n".join(text)
    print(
        version,
        "footer=", "issue(s) found" in bundle,
        "zero=", "No security issues found" in bundle,
        "format=", "--format" in bundle or "format?: ScanFormat" in bundle,
    )
PY