55 lines
2.1 KiB
Python
55 lines
2.1 KiB
Python
"""Sensitive-information redaction for knowledge content (phase 3).
|
|
|
|
Best-effort masking of secrets / PII before a note is persisted: API keys,
|
|
bearer tokens, generic ``key=secret`` assignments, Chinese mobile numbers,
|
|
ID-card numbers and email addresses. Conservative by design — it favours not
|
|
mangling legitimate prose over catching every possible secret.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
_MASK = "[REDACTED]"
|
|
|
|
# Order matters: more specific patterns first.
|
|
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|
("openai_key", re.compile(r"\bsk-[A-Za-z0-9]{20,}\b")),
|
|
("aws_key", re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b")),
|
|
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b")),
|
|
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]{16,}")),
|
|
# key/secret/password/token assignment: keep the field name, mask the value.
|
|
("assignment", re.compile(r"(?i)\b(api[_-]?key|secret|password|passwd|token|access[_-]?token)\b\s*[:=]\s*[\"']?[^\s\"']{6,}[\"']?")),
|
|
("cn_id_card", re.compile(r"\b\d{17}[\dXx]\b")),
|
|
("cn_mobile", re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)")),
|
|
("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")),
|
|
]
|
|
|
|
|
|
def redact(text: str | None) -> tuple[str, list[str]]:
|
|
"""Return ``(redacted_text, hit_kinds)``.
|
|
|
|
``hit_kinds`` lists which detectors fired (for logging/audit). When ``text``
|
|
is falsy it is returned unchanged with no hits.
|
|
"""
|
|
if not text:
|
|
return text or "", []
|
|
hits: list[str] = []
|
|
out = text
|
|
for kind, pattern in _PATTERNS:
|
|
def _sub(m: re.Match[str], _kind: str = kind) -> str:
|
|
# For assignments keep the field name + delimiter, mask only the value.
|
|
if _kind == "assignment":
|
|
head = re.split(r"[:=]", m.group(0), maxsplit=1)[0]
|
|
sep = ":" if ":" in m.group(0) else "="
|
|
return f"{head}{sep} {_MASK}"
|
|
if _kind == "bearer":
|
|
return f"Bearer {_MASK}"
|
|
return _MASK
|
|
|
|
new_out, n = pattern.subn(_sub, out)
|
|
if n:
|
|
hits.append(kind)
|
|
out = new_out
|
|
return out, hits
|