[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6J4TSKC110S4PXPYB4JQJST":3,"skill-same:01M6J4TSKC110S4PXPYB4JQJST":101},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":15,"licenseFileUrl":6,"sourceHtmlUrl":16,"categories":17,"keywords":23,"publishedAt":44,"linkOnly":45,"markdownContent":46,"markdownBody":47,"frontMatter":48,"refName":56,"commitSha":57,"skillDirectory":58,"forksCount":59,"repositoryHtmlUrl":60,"repositoryHomepageUrl":61,"ownerHtmlUrl":62,"ownerBlogUrl":6,"ownerDescription":63,"updatedAt":64,"install":65,"resources":78},"01M6J4TSKC110S4PXPYB4JQJST","pii-detection-pipeline",null,"Build automated PII detection and redaction pipelines using spaCy NER, Microsoft Presidio, and AWS Macie integration. Includes confidence scoring, custom entity type definitions, batch processing workflows, and multi-format document scanning for structured and unstructured data sources.","onfire7777\u002Funiversal-ai-skills-library","onfire7777","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F240866566?v=4","01M6HZJCF2QEW8DY7HE1JDKGK2","Jake B","01M6HZJCKK8SEXKN76V9SGKZ12",17,"Apache-2.0","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fpii-detection-pipeline\u002FSKILL.md",[18],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},"01JBSEEDCAT000000000000023","Skill与智能体开发","SCENARIO","使用场景",[24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43],"pipeline","detection","pii detection pipeline","pii","document","for","ner","unstructured","multi","processing","sources","type","scoring","build","workflows","confidence scoring","scanning","macie","data","batch","2026-11-10 10:12:30",false,"---\nname: pii-detection-pipeline\nlicense: Apache-2.0\ndescription: Build automated PII detection and redaction pipelines using spaCy NER, Microsoft Presidio, and AWS Macie integration. Includes confidence scoring, custom entity type definitions, batch processing workflows, and multi-format document scanning for structured and unstructured data sources.\nmetadata:\n  metadata:\n    author: mukul975\n    version: '1.0'\n    domain: privacy\n    subdomain: privacy-engineering\n    tags: pii-detection, presidio, spacy-ner, data-redaction, aws-macie\n---\n# Automated PII Detection and Redaction Pipeline\n\n## Overview\n\nAutomated PII detection is a foundational capability for privacy engineering, enabling organizations to discover, classify, and protect personal data at scale. This skill covers building production-grade PII detection pipelines that combine rule-based pattern matching, machine learning-based Named Entity Recognition (NER), and cloud-native discovery services.\n\n## PII Entity Types Catalog\n\n### Direct Identifiers\n\n| Entity Type | Examples | Detection Method | Risk Level |\n|------------|---------|-----------------|------------|\n| PERSON_NAME | \"John Smith\", \"Maria Garcia\" | NER model | High |\n| EMAIL_ADDRESS | \"j.smith@cipherengineeringlabs.com\" | Regex pattern | High |\n| PHONE_NUMBER | \"+1-555-0123\", \"(555) 012-3456\" | Regex + validation | High |\n| SSN | \"123-45-6789\" | Regex + checksum | Critical |\n| PASSPORT_NUMBER | \"AB1234567\" | Regex per country format | Critical |\n| DRIVER_LICENSE | \"D123-4567-8901\" | Regex per state\u002Fcountry | Critical |\n| CREDIT_CARD | \"4111-1111-1111-1111\" | Regex + Luhn checksum | Critical |\n| IBAN | \"GB82 WEST 1234 5698 7654 32\" | Regex + modulo-97 check | High |\n| IP_ADDRESS | \"192.168.1.1\", \"2001:db8::1\" | Regex (IPv4\u002FIPv6) | Medium |\n| MAC_ADDRESS | \"00:1A:2B:3C:4D:5E\" | Regex pattern | Medium |\n\n### Quasi-Identifiers\n\n| Entity Type | Examples | Detection Method | Risk Level |\n|------------|---------|-----------------|------------|\n| DATE_OF_BIRTH | \"1990-01-15\", \"January 15, 1990\" | NER + date parsing | Medium |\n| POSTAL_CODE | \"10001\", \"SW1A 1AA\" | Regex per country | Medium |\n| AGE | \"35 years old\", \"age: 42\" | NER + context | Low-Medium |\n| GENDER | \"male\", \"female\", \"non-binary\" | Dictionary + context | Low |\n| NATIONALITY | \"British\", \"Japanese\" | Dictionary + NER | Low-Medium |\n| LOCATION | \"123 Main St\", \"New York\" | NER model | Medium |\n\n### Sensitive Categories\n\n| Entity Type | Examples | Detection Method | Risk Level |\n|------------|---------|-----------------|------------|\n| MEDICAL_RECORD | \"MRN: 12345678\" | Regex + context | Critical |\n| HEALTH_CONDITION | \"diabetes\", \"HIV positive\" | Medical NER + dictionary | Critical |\n| RELIGIOUS_BELIEF | \"Muslim\", \"Catholic\" | Dictionary + context | High |\n| POLITICAL_OPINION | \"Democratic Party member\" | Dictionary + context | High |\n| SEXUAL_ORIENTATION | \"gay\", \"bisexual\" | Dictionary + context | High |\n| BIOMETRIC_DATA | \"fingerprint hash: ...\" | Context + pattern | Critical |\n| GENETIC_DATA | \"BRCA1 positive\" | Medical dictionary | Critical |\n\n## Microsoft Presidio Implementation\n\n### Pipeline Architecture\n\n```\nInput Data --> Presidio Analyzer --> Detected Entities --> Presidio Anonymizer --> Redacted Output\n                    |                      |                      |\n                    v                      v                      v\n             +------------+        +-------------+        +---------------+\n             | Recognizers|        | Score Filter |        | Operators     |\n             | - Pattern  |        | (threshold)  |        | - Replace     |\n             | - NER      |        |              |        | - Redact      |\n             | - Custom   |        |              |        | - Hash        |\n             +------------+        +-------------+        | - Mask        |\n                                                          | - Encrypt     |\n                                                          +---------------+\n```\n\n### Core Implementation\n\n```python\n\"\"\"\nPII detection and redaction pipeline using Microsoft Presidio.\nSupports structured and unstructured text with configurable\nentity types, confidence thresholds, and redaction strategies.\n\"\"\"\n\nfrom presidio_analyzer import AnalyzerEngine, PatternRecognizer, Pattern\nfrom presidio_analyzer.nlp_engine import NlpEngineProvider\nfrom presidio_anonymizer import AnonymizerEngine\nfrom presidio_anonymizer.entities import OperatorConfig\nimport json\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass DetectionResult:\n    entity_type: str\n    text: str\n    start: int\n    end: int\n    score: float\n    source: str\n\n\n@dataclass\nclass PipelineConfig:\n    language: str = \"en\"\n    score_threshold: float = 0.5\n    entities_to_detect: list[str] = field(default_factory=lambda: [\n        \"PERSON\", \"EMAIL_ADDRESS\", \"PHONE_NUMBER\", \"CREDIT_CARD\",\n        \"US_SSN\", \"US_DRIVER_LICENSE\", \"IBAN_CODE\", \"IP_ADDRESS\",\n        \"LOCATION\", \"DATE_TIME\", \"NRP\", \"MEDICAL_LICENSE\",\n        \"US_PASSPORT\", \"US_BANK_NUMBER\", \"UK_NHS\"\n    ])\n    redaction_strategy: str = \"replace\"  # replace, redact, hash, mask, encrypt\n\n\nclass PIIDetectionPipeline:\n    \"\"\"\n    Production PII detection pipeline built on Microsoft Presidio.\n    Supports custom entity types and configurable redaction strategies.\n    \"\"\"\n\n    def __init__(self, config: PipelineConfig):\n        self.config = config\n\n        # Initialize NLP engine with spaCy\n        nlp_config = {\n            \"nlp_engine_name\": \"spacy\",\n            \"models\": [{\"lang_code\": config.language, \"model_name\": \"en_core_web_lg\"}]\n        }\n        nlp_engine = NlpEngineProvider(nlp_configuration=nlp_config).create_engine()\n\n        # Initialize analyzer\n        self.analyzer = AnalyzerEngine(nlp_engine=nlp_engine)\n\n        # Register custom recognizers\n        self._register_custom_recognizers()\n\n        # Initialize anonymizer\n        self.anonymizer = AnonymizerEngine()\n\n    def _register_custom_recognizers(self):\n        \"\"\"Register custom PII recognizers beyond built-in types.\"\"\"\n\n        # UK National Insurance Number\n        nino_pattern = Pattern(\n            name=\"uk_nino\",\n            regex=r\"\\b[A-CEGHJ-PR-TW-Z]{2}\\s?\\d{2}\\s?\\d{2}\\s?\\d{2}\\s?[A-D]\\b\",\n            score=0.85\n        )\n        nino_recognizer = PatternRecognizer(\n            supported_entity=\"UK_NINO\",\n            patterns=[nino_pattern],\n            supported_language=\"en\"\n        )\n        self.analyzer.registry.add_recognizer(nino_recognizer)\n\n        # Vehicle Registration Number (UK format)\n        vrn_pattern = Pattern(\n            name=\"uk_vrn\",\n            regex=r\"\\b[A-Z]{2}\\d{2}\\s?[A-Z]{3}\\b\",\n            score=0.6\n        )\n        vrn_recognizer = PatternRecognizer(\n            supported_entity=\"UK_VRN\",\n            patterns=[vrn_pattern],\n            supported_language=\"en\"\n        )\n        self.analyzer.registry.add_recognizer(vrn_recognizer)\n\n        # Employee ID (Cipher Engineering Labs format: CEL-XXXXX)\n        emp_id_pattern = Pattern(\n            name=\"employee_id\",\n            regex=r\"\\bCEL-\\d{5}\\b\",\n            score=0.95\n        )\n        emp_id_recognizer = PatternRecognizer(\n            supported_entity=\"EMPLOYEE_ID\",\n            patterns=[emp_id_pattern],\n            supported_language=\"en\"\n        )\n        self.analyzer.registry.add_recognizer(emp_id_recognizer)\n\n    def detect(self, text: str) -> list[DetectionResult]:\n        \"\"\"\n        Detect PII entities in text.\n\n        Args:\n            text: Input text to scan\n\n        Returns:\n            List of detected PII entities with confidence scores\n        \"\"\"\n        results = self.analyzer.analyze(\n            text=text,\n            entities=self.config.entities_to_detect,\n            language=self.config.language,\n            score_threshold=self.config.score_threshold\n        )\n\n        return [\n            DetectionResult(\n                entity_type=r.entity_type,\n                text=text[r.start:r.end],\n                start=r.start,\n                end=r.end,\n                score=r.score,\n                source=r.analysis_explanation.recognizer if r.analysis_explanation else \"unknown\"\n            )\n            for r in results\n        ]\n\n    def redact(self, text: str) -> tuple[str, list[DetectionResult]]:\n        \"\"\"\n        Detect and redact PII from text.\n\n        Returns:\n            Tuple of (redacted_text, list_of_detections)\n        \"\"\"\n        # Detect entities\n        analyzer_results = self.analyzer.analyze(\n            text=text,\n            entities=self.config.entities_to_detect,\n            language=self.config.language,\n            score_threshold=self.config.score_threshold\n        )\n\n        # Configure redaction operator\n        operators = self._get_operators()\n\n        # Apply redaction\n        anonymized = self.anonymizer.anonymize(\n            text=text,\n            analyzer_results=analyzer_results,\n            operators=operators\n        )\n\n        detections = [\n            DetectionResult(\n                entity_type=r.entity_type,\n                text=text[r.start:r.end],\n                start=r.start,\n                end=r.end,\n                score=r.score,\n                source=\"presidio\"\n            )\n            for r in analyzer_results\n        ]\n\n        return anonymized.text, detections\n\n    def _get_operators(self) -> dict:\n        \"\"\"Configure anonymization operators based on strategy.\"\"\"\n        if self.config.redaction_strategy == \"replace\":\n            return {\"DEFAULT\": OperatorConfig(\"replace\", {\"new_value\": \"\u003CREDACTED>\"})}\n        elif self.config.redaction_strategy == \"hash\":\n            return {\"DEFAULT\": OperatorConfig(\"hash\", {\"hash_type\": \"sha256\"})}\n        elif self.config.redaction_strategy == \"mask\":\n            return {\"DEFAULT\": OperatorConfig(\"mask\", {\n                \"type\": \"mask\",\n                \"masking_char\": \"*\",\n                \"chars_to_mask\": 100,\n                \"from_end\": False\n            })}\n        elif self.config.redaction_strategy == \"redact\":\n            return {\"DEFAULT\": OperatorConfig(\"redact\", {})}\n        else:\n            return {\"DEFAULT\": OperatorConfig(\"replace\", {\"new_value\": \"\u003CREDACTED>\"})}\n\n\nclass BatchPIIProcessor:\n    \"\"\"\n    Process large volumes of documents for PII detection.\n    Supports structured (CSV\u002FJSON) and unstructured (text) formats.\n    \"\"\"\n\n    def __init__(self, pipeline: PIIDetectionPipeline):\n        self.pipeline = pipeline\n\n    def process_csv(\n        self,\n        input_path: str,\n        output_path: str,\n        columns_to_scan: list[str] = None\n    ) -> dict:\n        \"\"\"\n        Scan and redact PII in a CSV file.\n\n        Args:\n            input_path: Path to input CSV\n            output_path: Path to write redacted CSV\n            columns_to_scan: Columns to scan (all if None)\n\n        Returns:\n            Summary statistics of detections\n        \"\"\"\n        import pandas as pd\n\n        df = pd.read_csv(input_path)\n        stats = {\"total_rows\": len(df), \"detections\": {}, \"columns_scanned\": []}\n\n        scan_columns = columns_to_scan or df.columns.tolist()\n\n        for col in scan_columns:\n            if col not in df.columns:\n                continue\n\n            stats[\"columns_scanned\"].append(col)\n            col_detections = []\n\n            for idx, value in df[col].items():\n                if pd.isna(value):\n                    continue\n\n                text = str(value)\n                redacted, detections = self.pipeline.redact(text)\n                df.at[idx, col] = redacted\n\n                for d in detections:\n                    col_detections.append(d.entity_type)\n\n            # Count detection types for this column\n            for entity_type in set(col_detections):\n                count = col_detections.count(entity_type)\n                key = f\"{col}:{entity_type}\"\n                stats[\"detections\"][key] = count\n\n        df.to_csv(output_path, index=False)\n        stats[\"total_detections\"] = sum(stats[\"detections\"].values())\n\n        return stats\n\n    def process_text_files(\n        self,\n        file_paths: list[str],\n        output_dir: str\n    ) -> dict:\n        \"\"\"\n        Batch process text files for PII detection and redaction.\n\n        Returns summary statistics.\n        \"\"\"\n        import os\n\n        stats = {\n            \"files_processed\": 0,\n            \"total_detections\": 0,\n            \"entity_counts\": {},\n            \"high_risk_files\": []\n        }\n\n        for file_path in file_paths:\n            with open(file_path, \"r\", encoding=\"utf-8\") as f:\n                text = f.read()\n\n            redacted, detections = self.pipeline.redact(text)\n\n            # Write redacted output\n            output_path = os.path.join(output_dir, os.path.basename(file_path))\n            with open(output_path, \"w\", encoding=\"utf-8\") as f:\n                f.write(redacted)\n\n            # Update statistics\n            stats[\"files_processed\"] += 1\n            stats[\"total_detections\"] += len(detections)\n\n            for d in detections:\n                stats[\"entity_counts\"][d.entity_type] = (\n                    stats[\"entity_counts\"].get(d.entity_type, 0) + 1\n                )\n\n            # Flag high-risk files (contain critical PII)\n            critical_types = {\"US_SSN\", \"CREDIT_CARD\", \"MEDICAL_LICENSE\", \"UK_NHS\"}\n            if any(d.entity_type in critical_types for d in detections):\n                stats[\"high_risk_files\"].append(file_path)\n\n        return stats\n```\n\n## spaCy NER Custom Training\n\n### Training Custom Entity Types\n\n```python\n\"\"\"\nTrain custom spaCy NER model for domain-specific PII entities.\n\"\"\"\n\nimport spacy\nfrom spacy.training import Example\nimport random\n\n\ndef create_training_data() -> list[tuple[str, dict]]:\n    \"\"\"\n    Create training examples for custom PII entity types.\n\n    Returns list of (text, annotations) tuples.\n    \"\"\"\n    training_data = [\n        (\n            \"Patient MRN 12345678 was admitted on 2024-01-15\",\n            {\"entities\": [(12, 20, \"MEDICAL_RECORD_NUMBER\")]}\n        ),\n        (\n            \"Employee CEL-00142 reported the incident\",\n            {\"entities\": [(9, 18, \"EMPLOYEE_ID\")]}\n        ),\n        (\n            \"Policy holder number PLH-2024-99887 filed a claim\",\n            {\"entities\": [(22, 35, \"POLICY_NUMBER\")]}\n        ),\n        (\n            \"The customer with loyalty ID LYL-A1B2C3 requested data export\",\n            {\"entities\": [(29, 39, \"LOYALTY_ID\")]}\n        ),\n    ]\n    return training_data\n\n\ndef train_custom_ner(\n    base_model: str = \"en_core_web_lg\",\n    training_data: list = None,\n    n_iter: int = 30,\n    output_dir: str = \".\u002Fcustom_ner_model\"\n):\n    \"\"\"\n    Fine-tune spaCy NER model with custom PII entity types.\n\n    Args:\n        base_model: Base spaCy model to fine-tune\n        training_data: List of (text, annotations) tuples\n        n_iter: Number of training iterations\n        output_dir: Directory to save trained model\n    \"\"\"\n    if training_data is None:\n        training_data = create_training_data()\n\n    nlp = spacy.load(base_model)\n\n    # Get or create NER pipe\n    if \"ner\" not in nlp.pipe_names:\n        ner = nlp.add_pipe(\"ner\", last=True)\n    else:\n        ner = nlp.get_pipe(\"ner\")\n\n    # Add custom entity labels\n    custom_labels = set()\n    for _, annotations in training_data:\n        for ent in annotations.get(\"entities\", []):\n            custom_labels.add(ent[2])\n\n    for label in custom_labels:\n        ner.add_label(label)\n\n    # Train\n    optimizer = nlp.resume_training()\n    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != \"ner\"]\n\n    with nlp.disable_pipes(*other_pipes):\n        for iteration in range(n_iter):\n            random.shuffle(training_data)\n            losses = {}\n\n            for text, annotations in training_data:\n                doc = nlp.make_doc(text)\n                example = Example.from_dict(doc, annotations)\n                nlp.update([example], drop=0.35, sgd=optimizer, losses=losses)\n\n            if iteration % 10 == 0:\n                print(f\"Iteration {iteration}, Losses: {losses}\")\n\n    nlp.to_disk(output_dir)\n    return nlp\n```\n\n## AWS Macie Integration\n\n### Architecture\n\n```\nS3 Buckets --> Macie Classification Jobs --> Findings --> EventBridge --> Lambda\n                                                |                          |\n                                                v                          v\n                                        Security Hub              Remediation\n                                        (centralized)             - Tag sensitive\n                                                                  - Encrypt\n                                                                  - Notify owner\n                                                                  - Quarantine\n```\n\n### Macie Job Configuration\n\n```python\n\"\"\"\nAWS Macie integration for cloud-native PII detection in S3 buckets.\n\"\"\"\n\nimport boto3\nfrom datetime import datetime\n\n\nclass MacieIntegration:\n    \"\"\"\n    Configure and manage AWS Macie classification jobs for\n    automated PII detection across S3 data stores.\n    \"\"\"\n\n    def __init__(self, region: str = \"us-east-1\"):\n        self.macie_client = boto3.client(\"macie2\", region_name=region)\n\n    def create_classification_job(\n        self,\n        bucket_name: str,\n        job_name: str,\n        custom_data_identifiers: list[str] = None,\n        schedule: str = \"ONE_TIME\"\n    ) -> str:\n        \"\"\"\n        Create a Macie classification job for an S3 bucket.\n\n        Args:\n            bucket_name: Target S3 bucket\n            job_name: Descriptive job name\n            custom_data_identifiers: IDs of custom data identifier resources\n            schedule: ONE_TIME or SCHEDULED\n\n        Returns:\n            Job ID\n        \"\"\"\n        job_config = {\n            \"name\": job_name,\n            \"description\": f\"PII detection scan for {bucket_name}\",\n            \"jobType\": schedule,\n            \"s3JobDefinition\": {\n                \"bucketDefinitions\": [\n                    {\n                        \"accountId\": self._get_account_id(),\n                        \"buckets\": [bucket_name]\n                    }\n                ],\n                \"scoping\": {\n                    \"includes\": {\n                        \"and\": [\n                            {\n                                \"simpleScopeTerm\": {\n                                    \"comparator\": \"STARTS_WITH\",\n                                    \"key\": \"OBJECT_EXTENSION\",\n                                    \"values\": [\"csv\", \"json\", \"txt\", \"pdf\", \"docx\", \"xlsx\", \"parquet\"]\n                                }\n                            }\n                        ]\n                    }\n                }\n            },\n            \"managedDataIdentifierSelector\": \"ALL\",\n            \"tags\": {\n                \"Team\": \"Privacy Engineering\",\n                \"Purpose\": \"PII Detection\"\n            }\n        }\n\n        if custom_data_identifiers:\n            job_config[\"customDataIdentifierIds\"] = custom_data_identifiers\n\n        response = self.macie_client.create_classification_job(**job_config)\n        return response[\"jobId\"]\n\n    def create_custom_data_identifier(\n        self,\n        name: str,\n        description: str,\n        regex: str,\n        keywords: list[str] = None,\n        maximum_match_distance: int = 50\n    ) -> str:\n        \"\"\"\n        Create a custom data identifier for organization-specific PII patterns.\n\n        Returns:\n            Custom data identifier ID\n        \"\"\"\n        params = {\n            \"name\": name,\n            \"description\": description,\n            \"regex\": regex,\n            \"maximumMatchDistance\": maximum_match_distance\n        }\n\n        if keywords:\n            params[\"keywords\"] = keywords\n\n        response = self.macie_client.create_custom_data_identifier(**params)\n        return response[\"customDataIdentifierId\"]\n\n    def _get_account_id(self) -> str:\n        \"\"\"Get the current AWS account ID.\"\"\"\n        sts = boto3.client(\"sts\")\n        return sts.get_caller_identity()[\"Account\"]\n\n    def get_findings_summary(self, job_id: str) -> dict:\n        \"\"\"Get summary of findings from a classification job.\"\"\"\n        response = self.macie_client.list_findings(\n            findingCriteria={\n                \"criterion\": {\n                    \"classificationDetails.jobId\": {\n                        \"eq\": [job_id]\n                    }\n                }\n            }\n        )\n\n        findings = []\n        if response[\"findingIds\"]:\n            details = self.macie_client.get_findings(findingIds=response[\"findingIds\"])\n            findings = details[\"findings\"]\n\n        summary = {\n            \"total_findings\": len(findings),\n            \"severity_counts\": {},\n            \"entity_type_counts\": {},\n            \"affected_objects\": []\n        }\n\n        for finding in findings:\n            severity = finding.get(\"severity\", {}).get(\"description\", \"unknown\")\n            summary[\"severity_counts\"][severity] = (\n                summary[\"severity_counts\"].get(severity, 0) + 1\n            )\n\n            sensitive_data = finding.get(\"classificationDetails\", {}).get(\n                \"result\", {}\n            ).get(\"sensitiveData\", [])\n\n            for sd in sensitive_data:\n                category = sd.get(\"category\", \"unknown\")\n                summary[\"entity_type_counts\"][category] = (\n                    summary[\"entity_type_counts\"].get(category, 0)\n                    + sd.get(\"totalCount\", 0)\n                )\n\n            resource = finding.get(\"resourcesAffected\", {}).get(\"s3Object\", {})\n            if resource:\n                summary[\"affected_objects\"].append(resource.get(\"key\", \"unknown\"))\n\n        return summary\n```\n\n## Confidence Scoring Framework\n\n| Score Range | Confidence Level | Recommended Action |\n|-------------|-----------------|-------------------|\n| 0.95 - 1.00 | Very High | Auto-redact |\n| 0.80 - 0.94 | High | Auto-redact with logging |\n| 0.60 - 0.79 | Medium | Flag for human review |\n| 0.40 - 0.59 | Low | Log only, no action |\n| 0.00 - 0.39 | Very Low | Ignore |\n\n## References\n\n- Microsoft Presidio Documentation: microsoft.github.io\u002Fpresidio\n- spaCy NER Documentation: spacy.io\u002Fusage\u002Flinguistic-features#named-entities\n- AWS Macie Documentation: docs.aws.amazon.com\u002Fmacie\n- Google Cloud DLP API Documentation\n- NIST SP 800-188 — De-Identifying Government Datasets\n- Article 29 WP Opinion 05\u002F2014 on Anonymisation Techniques\n","# Automated PII Detection and Redaction Pipeline\n\n## Overview\n\nAutomated PII detection is a foundational capability for privacy engineering, enabling organizations to discover, classify, and protect personal data at scale. This skill covers building production-grade PII detection pipelines that combine rule-based pattern matching, machine learning-based Named Entity Recognition (NER), and cloud-native discovery services.\n\n## PII Entity Types Catalog\n\n### Direct Identifiers\n\n| Entity Type | Examples | Detection Method | Risk Level |\n|------------|---------|-----------------|------------|\n| PERSON_NAME | \"John Smith\", \"Maria Garcia\" | NER model | High |\n| EMAIL_ADDRESS | \"j.smith@cipherengineeringlabs.com\" | Regex pattern | High |\n| PHONE_NUMBER | \"+1-555-0123\", \"(555) 012-3456\" | Regex + validation | High |\n| SSN | \"123-45-6789\" | Regex + checksum | Critical |\n| PASSPORT_NUMBER | \"AB1234567\" | Regex per country format | Critical |\n| DRIVER_LICENSE | \"D123-4567-8901\" | Regex per state\u002Fcountry | Critical |\n| CREDIT_CARD | \"4111-1111-1111-1111\" | Regex + Luhn checksum | Critical |\n| IBAN | \"GB82 WEST 1234 5698 7654 32\" | Regex + modulo-97 check | High |\n| IP_ADDRESS | \"192.168.1.1\", \"2001:db8::1\" | Regex (IPv4\u002FIPv6) | Medium |\n| MAC_ADDRESS | \"00:1A:2B:3C:4D:5E\" | Regex pattern | Medium |\n\n### Quasi-Identifiers\n\n| Entity Type | Examples | Detection Method | Risk Level |\n|------------|---------|-----------------|------------|\n| DATE_OF_BIRTH | \"1990-01-15\", \"January 15, 1990\" | NER + date parsing | Medium |\n| POSTAL_CODE | \"10001\", \"SW1A 1AA\" | Regex per country | Medium |\n| AGE | \"35 years old\", \"age: 42\" | NER + context | Low-Medium |\n| GENDER | \"male\", \"female\", \"non-binary\" | Dictionary + context | Low |\n| NATIONALITY | \"British\", \"Japanese\" | Dictionary + NER | Low-Medium |\n| LOCATION | \"123 Main St\", \"New York\" | NER model | Medium |\n\n### Sensitive Categories\n\n| Entity Type | Examples | Detection Method | Risk Level |\n|------------|---------|-----------------|------------|\n| MEDICAL_RECORD | \"MRN: 12345678\" | Regex + context | Critical |\n| HEALTH_CONDITION | \"diabetes\", \"HIV positive\" | Medical NER + dictionary | Critical |\n| RELIGIOUS_BELIEF | \"Muslim\", \"Catholic\" | Dictionary + context | High |\n| POLITICAL_OPINION | \"Democratic Party member\" | Dictionary + context | High |\n| SEXUAL_ORIENTATION | \"gay\", \"bisexual\" | Dictionary + context | High |\n| BIOMETRIC_DATA | \"fingerprint hash: ...\" | Context + pattern | Critical |\n| GENETIC_DATA | \"BRCA1 positive\" | Medical dictionary | Critical |\n\n## Microsoft Presidio Implementation\n\n### Pipeline Architecture\n\n```\nInput Data --> Presidio Analyzer --> Detected Entities --> Presidio Anonymizer --> Redacted Output\n                    |                      |                      |\n                    v                      v                      v\n             +------------+        +-------------+        +---------------+\n             | Recognizers|        | Score Filter |        | Operators     |\n             | - Pattern  |        | (threshold)  |        | - Replace     |\n             | - NER      |        |              |        | - Redact      |\n             | - Custom   |        |              |        | - Hash        |\n             +------------+        +-------------+        | - Mask        |\n                                                          | - Encrypt     |\n                                                          +---------------+\n```\n\n### Core Implementation\n\n```python\n\"\"\"\nPII detection and redaction pipeline using Microsoft Presidio.\nSupports structured and unstructured text with configurable\nentity types, confidence thresholds, and redaction strategies.\n\"\"\"\n\nfrom presidio_analyzer import AnalyzerEngine, PatternRecognizer, Pattern\nfrom presidio_analyzer.nlp_engine import NlpEngineProvider\nfrom presidio_anonymizer import AnonymizerEngine\nfrom presidio_anonymizer.entities import OperatorConfig\nimport json\nfrom dataclasses import dataclass, field\n\n\n@dataclass\nclass DetectionResult:\n    entity_type: str\n    text: str\n    start: int\n    end: int\n    score: float\n    source: str\n\n\n@dataclass\nclass PipelineConfig:\n    language: str = \"en\"\n    score_threshold: float = 0.5\n    entities_to_detect: list[str] = field(default_factory=lambda: [\n        \"PERSON\", \"EMAIL_ADDRESS\", \"PHONE_NUMBER\", \"CREDIT_CARD\",\n        \"US_SSN\", \"US_DRIVER_LICENSE\", \"IBAN_CODE\", \"IP_ADDRESS\",\n        \"LOCATION\", \"DATE_TIME\", \"NRP\", \"MEDICAL_LICENSE\",\n        \"US_PASSPORT\", \"US_BANK_NUMBER\", \"UK_NHS\"\n    ])\n    redaction_strategy: str = \"replace\"  # replace, redact, hash, mask, encrypt\n\n\nclass PIIDetectionPipeline:\n    \"\"\"\n    Production PII detection pipeline built on Microsoft Presidio.\n    Supports custom entity types and configurable redaction strategies.\n    \"\"\"\n\n    def __init__(self, config: PipelineConfig):\n        self.config = config\n\n        # Initialize NLP engine with spaCy\n        nlp_config = {\n            \"nlp_engine_name\": \"spacy\",\n            \"models\": [{\"lang_code\": config.language, \"model_name\": \"en_core_web_lg\"}]\n        }\n        nlp_engine = NlpEngineProvider(nlp_configuration=nlp_config).create_engine()\n\n        # Initialize analyzer\n        self.analyzer = AnalyzerEngine(nlp_engine=nlp_engine)\n\n        # Register custom recognizers\n        self._register_custom_recognizers()\n\n        # Initialize anonymizer\n        self.anonymizer = AnonymizerEngine()\n\n    def _register_custom_recognizers(self):\n        \"\"\"Register custom PII recognizers beyond built-in types.\"\"\"\n\n        # UK National Insurance Number\n        nino_pattern = Pattern(\n            name=\"uk_nino\",\n            regex=r\"\\b[A-CEGHJ-PR-TW-Z]{2}\\s?\\d{2}\\s?\\d{2}\\s?\\d{2}\\s?[A-D]\\b\",\n            score=0.85\n        )\n        nino_recognizer = PatternRecognizer(\n            supported_entity=\"UK_NINO\",\n            patterns=[nino_pattern],\n            supported_language=\"en\"\n        )\n        self.analyzer.registry.add_recognizer(nino_recognizer)\n\n        # Vehicle Registration Number (UK format)\n        vrn_pattern = Pattern(\n            name=\"uk_vrn\",\n            regex=r\"\\b[A-Z]{2}\\d{2}\\s?[A-Z]{3}\\b\",\n            score=0.6\n        )\n        vrn_recognizer = PatternRecognizer(\n            supported_entity=\"UK_VRN\",\n            patterns=[vrn_pattern],\n            supported_language=\"en\"\n        )\n        self.analyzer.registry.add_recognizer(vrn_recognizer)\n\n        # Employee ID (Cipher Engineering Labs format: CEL-XXXXX)\n        emp_id_pattern = Pattern(\n            name=\"employee_id\",\n            regex=r\"\\bCEL-\\d{5}\\b\",\n            score=0.95\n        )\n        emp_id_recognizer = PatternRecognizer(\n            supported_entity=\"EMPLOYEE_ID\",\n            patterns=[emp_id_pattern],\n            supported_language=\"en\"\n        )\n        self.analyzer.registry.add_recognizer(emp_id_recognizer)\n\n    def detect(self, text: str) -> list[DetectionResult]:\n        \"\"\"\n        Detect PII entities in text.\n\n        Args:\n            text: Input text to scan\n\n        Returns:\n            List of detected PII entities with confidence scores\n        \"\"\"\n        results = self.analyzer.analyze(\n            text=text,\n            entities=self.config.entities_to_detect,\n            language=self.config.language,\n            score_threshold=self.config.score_threshold\n        )\n\n        return [\n            DetectionResult(\n                entity_type=r.entity_type,\n                text=text[r.start:r.end],\n                start=r.start,\n                end=r.end,\n                score=r.score,\n                source=r.analysis_explanation.recognizer if r.analysis_explanation else \"unknown\"\n            )\n            for r in results\n        ]\n\n    def redact(self, text: str) -> tuple[str, list[DetectionResult]]:\n        \"\"\"\n        Detect and redact PII from text.\n\n        Returns:\n            Tuple of (redacted_text, list_of_detections)\n        \"\"\"\n        # Detect entities\n        analyzer_results = self.analyzer.analyze(\n            text=text,\n            entities=self.config.entities_to_detect,\n            language=self.config.language,\n            score_threshold=self.config.score_threshold\n        )\n\n        # Configure redaction operator\n        operators = self._get_operators()\n\n        # Apply redaction\n        anonymized = self.anonymizer.anonymize(\n            text=text,\n            analyzer_results=analyzer_results,\n            operators=operators\n        )\n\n        detections = [\n            DetectionResult(\n                entity_type=r.entity_type,\n                text=text[r.start:r.end],\n                start=r.start,\n                end=r.end,\n                score=r.score,\n                source=\"presidio\"\n            )\n            for r in analyzer_results\n        ]\n\n        return anonymized.text, detections\n\n    def _get_operators(self) -> dict:\n        \"\"\"Configure anonymization operators based on strategy.\"\"\"\n        if self.config.redaction_strategy == \"replace\":\n            return {\"DEFAULT\": OperatorConfig(\"replace\", {\"new_value\": \"\u003CREDACTED>\"})}\n        elif self.config.redaction_strategy == \"hash\":\n            return {\"DEFAULT\": OperatorConfig(\"hash\", {\"hash_type\": \"sha256\"})}\n        elif self.config.redaction_strategy == \"mask\":\n            return {\"DEFAULT\": OperatorConfig(\"mask\", {\n                \"type\": \"mask\",\n                \"masking_char\": \"*\",\n                \"chars_to_mask\": 100,\n                \"from_end\": False\n            })}\n        elif self.config.redaction_strategy == \"redact\":\n            return {\"DEFAULT\": OperatorConfig(\"redact\", {})}\n        else:\n            return {\"DEFAULT\": OperatorConfig(\"replace\", {\"new_value\": \"\u003CREDACTED>\"})}\n\n\nclass BatchPIIProcessor:\n    \"\"\"\n    Process large volumes of documents for PII detection.\n    Supports structured (CSV\u002FJSON) and unstructured (text) formats.\n    \"\"\"\n\n    def __init__(self, pipeline: PIIDetectionPipeline):\n        self.pipeline = pipeline\n\n    def process_csv(\n        self,\n        input_path: str,\n        output_path: str,\n        columns_to_scan: list[str] = None\n    ) -> dict:\n        \"\"\"\n        Scan and redact PII in a CSV file.\n\n        Args:\n            input_path: Path to input CSV\n            output_path: Path to write redacted CSV\n            columns_to_scan: Columns to scan (all if None)\n\n        Returns:\n            Summary statistics of detections\n        \"\"\"\n        import pandas as pd\n\n        df = pd.read_csv(input_path)\n        stats = {\"total_rows\": len(df), \"detections\": {}, \"columns_scanned\": []}\n\n        scan_columns = columns_to_scan or df.columns.tolist()\n\n        for col in scan_columns:\n            if col not in df.columns:\n                continue\n\n            stats[\"columns_scanned\"].append(col)\n            col_detections = []\n\n            for idx, value in df[col].items():\n                if pd.isna(value):\n                    continue\n\n                text = str(value)\n                redacted, detections = self.pipeline.redact(text)\n                df.at[idx, col] = redacted\n\n                for d in detections:\n                    col_detections.append(d.entity_type)\n\n            # Count detection types for this column\n            for entity_type in set(col_detections):\n                count = col_detections.count(entity_type)\n                key = f\"{col}:{entity_type}\"\n                stats[\"detections\"][key] = count\n\n        df.to_csv(output_path, index=False)\n        stats[\"total_detections\"] = sum(stats[\"detections\"].values())\n\n        return stats\n\n    def process_text_files(\n        self,\n        file_paths: list[str],\n        output_dir: str\n    ) -> dict:\n        \"\"\"\n        Batch process text files for PII detection and redaction.\n\n        Returns summary statistics.\n        \"\"\"\n        import os\n\n        stats = {\n            \"files_processed\": 0,\n            \"total_detections\": 0,\n            \"entity_counts\": {},\n            \"high_risk_files\": []\n        }\n\n        for file_path in file_paths:\n            with open(file_path, \"r\", encoding=\"utf-8\") as f:\n                text = f.read()\n\n            redacted, detections = self.pipeline.redact(text)\n\n            # Write redacted output\n            output_path = os.path.join(output_dir, os.path.basename(file_path))\n            with open(output_path, \"w\", encoding=\"utf-8\") as f:\n                f.write(redacted)\n\n            # Update statistics\n            stats[\"files_processed\"] += 1\n            stats[\"total_detections\"] += len(detections)\n\n            for d in detections:\n                stats[\"entity_counts\"][d.entity_type] = (\n                    stats[\"entity_counts\"].get(d.entity_type, 0) + 1\n                )\n\n            # Flag high-risk files (contain critical PII)\n            critical_types = {\"US_SSN\", \"CREDIT_CARD\", \"MEDICAL_LICENSE\", \"UK_NHS\"}\n            if any(d.entity_type in critical_types for d in detections):\n                stats[\"high_risk_files\"].append(file_path)\n\n        return stats\n```\n\n## spaCy NER Custom Training\n\n### Training Custom Entity Types\n\n```python\n\"\"\"\nTrain custom spaCy NER model for domain-specific PII entities.\n\"\"\"\n\nimport spacy\nfrom spacy.training import Example\nimport random\n\n\ndef create_training_data() -> list[tuple[str, dict]]:\n    \"\"\"\n    Create training examples for custom PII entity types.\n\n    Returns list of (text, annotations) tuples.\n    \"\"\"\n    training_data = [\n        (\n            \"Patient MRN 12345678 was admitted on 2024-01-15\",\n            {\"entities\": [(12, 20, \"MEDICAL_RECORD_NUMBER\")]}\n        ),\n        (\n            \"Employee CEL-00142 reported the incident\",\n            {\"entities\": [(9, 18, \"EMPLOYEE_ID\")]}\n        ),\n        (\n            \"Policy holder number PLH-2024-99887 filed a claim\",\n            {\"entities\": [(22, 35, \"POLICY_NUMBER\")]}\n        ),\n        (\n            \"The customer with loyalty ID LYL-A1B2C3 requested data export\",\n            {\"entities\": [(29, 39, \"LOYALTY_ID\")]}\n        ),\n    ]\n    return training_data\n\n\ndef train_custom_ner(\n    base_model: str = \"en_core_web_lg\",\n    training_data: list = None,\n    n_iter: int = 30,\n    output_dir: str = \".\u002Fcustom_ner_model\"\n):\n    \"\"\"\n    Fine-tune spaCy NER model with custom PII entity types.\n\n    Args:\n        base_model: Base spaCy model to fine-tune\n        training_data: List of (text, annotations) tuples\n        n_iter: Number of training iterations\n        output_dir: Directory to save trained model\n    \"\"\"\n    if training_data is None:\n        training_data = create_training_data()\n\n    nlp = spacy.load(base_model)\n\n    # Get or create NER pipe\n    if \"ner\" not in nlp.pipe_names:\n        ner = nlp.add_pipe(\"ner\", last=True)\n    else:\n        ner = nlp.get_pipe(\"ner\")\n\n    # Add custom entity labels\n    custom_labels = set()\n    for _, annotations in training_data:\n        for ent in annotations.get(\"entities\", []):\n            custom_labels.add(ent[2])\n\n    for label in custom_labels:\n        ner.add_label(label)\n\n    # Train\n    optimizer = nlp.resume_training()\n    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != \"ner\"]\n\n    with nlp.disable_pipes(*other_pipes):\n        for iteration in range(n_iter):\n            random.shuffle(training_data)\n            losses = {}\n\n            for text, annotations in training_data:\n                doc = nlp.make_doc(text)\n                example = Example.from_dict(doc, annotations)\n                nlp.update([example], drop=0.35, sgd=optimizer, losses=losses)\n\n            if iteration % 10 == 0:\n                print(f\"Iteration {iteration}, Losses: {losses}\")\n\n    nlp.to_disk(output_dir)\n    return nlp\n```\n\n## AWS Macie Integration\n\n### Architecture\n\n```\nS3 Buckets --> Macie Classification Jobs --> Findings --> EventBridge --> Lambda\n                                                |                          |\n                                                v                          v\n                                        Security Hub              Remediation\n                                        (centralized)             - Tag sensitive\n                                                                  - Encrypt\n                                                                  - Notify owner\n                                                                  - Quarantine\n```\n\n### Macie Job Configuration\n\n```python\n\"\"\"\nAWS Macie integration for cloud-native PII detection in S3 buckets.\n\"\"\"\n\nimport boto3\nfrom datetime import datetime\n\n\nclass MacieIntegration:\n    \"\"\"\n    Configure and manage AWS Macie classification jobs for\n    automated PII detection across S3 data stores.\n    \"\"\"\n\n    def __init__(self, region: str = \"us-east-1\"):\n        self.macie_client = boto3.client(\"macie2\", region_name=region)\n\n    def create_classification_job(\n        self,\n        bucket_name: str,\n        job_name: str,\n        custom_data_identifiers: list[str] = None,\n        schedule: str = \"ONE_TIME\"\n    ) -> str:\n        \"\"\"\n        Create a Macie classification job for an S3 bucket.\n\n        Args:\n            bucket_name: Target S3 bucket\n            job_name: Descriptive job name\n            custom_data_identifiers: IDs of custom data identifier resources\n            schedule: ONE_TIME or SCHEDULED\n\n        Returns:\n            Job ID\n        \"\"\"\n        job_config = {\n            \"name\": job_name,\n            \"description\": f\"PII detection scan for {bucket_name}\",\n            \"jobType\": schedule,\n            \"s3JobDefinition\": {\n                \"bucketDefinitions\": [\n                    {\n                        \"accountId\": self._get_account_id(),\n                        \"buckets\": [bucket_name]\n                    }\n                ],\n                \"scoping\": {\n                    \"includes\": {\n                        \"and\": [\n                            {\n                                \"simpleScopeTerm\": {\n                                    \"comparator\": \"STARTS_WITH\",\n                                    \"key\": \"OBJECT_EXTENSION\",\n                                    \"values\": [\"csv\", \"json\", \"txt\", \"pdf\", \"docx\", \"xlsx\", \"parquet\"]\n                                }\n                            }\n                        ]\n                    }\n                }\n            },\n            \"managedDataIdentifierSelector\": \"ALL\",\n            \"tags\": {\n                \"Team\": \"Privacy Engineering\",\n                \"Purpose\": \"PII Detection\"\n            }\n        }\n\n        if custom_data_identifiers:\n            job_config[\"customDataIdentifierIds\"] = custom_data_identifiers\n\n        response = self.macie_client.create_classification_job(**job_config)\n        return response[\"jobId\"]\n\n    def create_custom_data_identifier(\n        self,\n        name: str,\n        description: str,\n        regex: str,\n        keywords: list[str] = None,\n        maximum_match_distance: int = 50\n    ) -> str:\n        \"\"\"\n        Create a custom data identifier for organization-specific PII patterns.\n\n        Returns:\n            Custom data identifier ID\n        \"\"\"\n        params = {\n            \"name\": name,\n            \"description\": description,\n            \"regex\": regex,\n            \"maximumMatchDistance\": maximum_match_distance\n        }\n\n        if keywords:\n            params[\"keywords\"] = keywords\n\n        response = self.macie_client.create_custom_data_identifier(**params)\n        return response[\"customDataIdentifierId\"]\n\n    def _get_account_id(self) -> str:\n        \"\"\"Get the current AWS account ID.\"\"\"\n        sts = boto3.client(\"sts\")\n        return sts.get_caller_identity()[\"Account\"]\n\n    def get_findings_summary(self, job_id: str) -> dict:\n        \"\"\"Get summary of findings from a classification job.\"\"\"\n        response = self.macie_client.list_findings(\n            findingCriteria={\n                \"criterion\": {\n                    \"classificationDetails.jobId\": {\n                        \"eq\": [job_id]\n                    }\n                }\n            }\n        )\n\n        findings = []\n        if response[\"findingIds\"]:\n            details = self.macie_client.get_findings(findingIds=response[\"findingIds\"])\n            findings = details[\"findings\"]\n\n        summary = {\n            \"total_findings\": len(findings),\n            \"severity_counts\": {},\n            \"entity_type_counts\": {},\n            \"affected_objects\": []\n        }\n\n        for finding in findings:\n            severity = finding.get(\"severity\", {}).get(\"description\", \"unknown\")\n            summary[\"severity_counts\"][severity] = (\n                summary[\"severity_counts\"].get(severity, 0) + 1\n            )\n\n            sensitive_data = finding.get(\"classificationDetails\", {}).get(\n                \"result\", {}\n            ).get(\"sensitiveData\", [])\n\n            for sd in sensitive_data:\n                category = sd.get(\"category\", \"unknown\")\n                summary[\"entity_type_counts\"][category] = (\n                    summary[\"entity_type_counts\"].get(category, 0)\n                    + sd.get(\"totalCount\", 0)\n                )\n\n            resource = finding.get(\"resourcesAffected\", {}).get(\"s3Object\", {})\n            if resource:\n                summary[\"affected_objects\"].append(resource.get(\"key\", \"unknown\"))\n\n        return summary\n```\n\n## Confidence Scoring Framework\n\n| Score Range | Confidence Level | Recommended Action |\n|-------------|-----------------|-------------------|\n| 0.95 - 1.00 | Very High | Auto-redact |\n| 0.80 - 0.94 | High | Auto-redact with logging |\n| 0.60 - 0.79 | Medium | Flag for human review |\n| 0.40 - 0.59 | Low | Log only, no action |\n| 0.00 - 0.39 | Very Low | Ignore |\n\n## References\n\n- Microsoft Presidio Documentation: microsoft.github.io\u002Fpresidio\n- spaCy NER Documentation: spacy.io\u002Fusage\u002Flinguistic-features#named-entities\n- AWS Macie Documentation: docs.aws.amazon.com\u002Fmacie\n- Google Cloud DLP API Documentation\n- NIST SP 800-188 — De-Identifying Government Datasets\n- Article 29 WP Opinion 05\u002F2014 on Anonymisation Techniques\n",{"name":5,"license":15,"metadata":49,"description":7},{"metadata":50},{"tags":51,"author":52,"domain":53,"version":54,"subdomain":55},"pii-detection, presidio, spacy-ner, data-redaction, aws-macie","mukul975","privacy","1.0","privacy-engineering","main","2a76bf42ce5c860adf8d0f3fcf3f6804109ac65b","skills\u002Fpii-detection-pipeline",0,"https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fwiki","https:\u002F\u002Fgithub.com\u002Fonfire7777","trying to create something actually useful","2026-09-11 13:25:26",{"source":8,"skillName":5,"commands":66,"prompt":76,"zipAvailable":77},[67,70,73],{"tool":68,"command":69},"npx","npx skills add onfire7777\u002Funiversal-ai-skills-library --skill pii-detection-pipeline",{"tool":71,"command":72},"bunx","bunx skills add onfire7777\u002Funiversal-ai-skills-library --skill pii-detection-pipeline",{"tool":74,"command":75},"pnpm","pnpm dlx skills add onfire7777\u002Funiversal-ai-skills-library --skill pii-detection-pipeline","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fpii-detection-pipeline\u002FSKILL.md",true,[79,85,91,95],{"path":80,"resourceType":81,"resourceTypeDesc":82,"sizeBytes":83,"sourceHtmlUrl":84},"assets\u002Ftemplate.md","ASSET","资产文件",1055,"https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fpii-detection-pipeline\u002Fassets\u002Ftemplate.md",{"path":86,"resourceType":87,"resourceTypeDesc":88,"sizeBytes":89,"sourceHtmlUrl":90},"references\u002Fstandards.md","REFERENCE","参考文档",942,"https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fpii-detection-pipeline\u002Freferences\u002Fstandards.md",{"path":92,"resourceType":87,"resourceTypeDesc":88,"sizeBytes":93,"sourceHtmlUrl":94},"references\u002Fworkflows.md",1527,"https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fpii-detection-pipeline\u002Freferences\u002Fworkflows.md",{"path":96,"resourceType":97,"resourceTypeDesc":98,"sizeBytes":99,"sourceHtmlUrl":100},"scripts\u002Fprocess.py","SCRIPT","脚本",6128,"https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fpii-detection-pipeline\u002Fscripts\u002Fprocess.py",[102,130,139,149,170,186],{"skillNo":103,"name":104,"nameZh":6,"description":105,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":106,"licenseFileUrl":6,"sourceHtmlUrl":107,"categories":108,"keywords":118,"publishedAt":44},"01M6J7MPK8ZE9VQJR4KK6ZMZJQ","zyte-api-automation","Automate Zyte API tasks via Rube MCP (Composio). Always search tools first for current schemas.","MIT","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fzyte-api-automation\u002FSKILL.md",[109,112,113],{"categoryNo":110,"categoryName":111,"taxonomyType":21,"taxonomyTypeDesc":22},"01JBSEEDCAT000000000000010","后端与接口",{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},{"categoryNo":114,"categoryName":115,"taxonomyType":116,"taxonomyTypeDesc":117},"01JBSEEDCAT000000000000030","开发工程师","PROFESSION","适用职业",[119,120,121,29,122,123,124,125,126,127,128,129],"automation","api","zyte api automation","MCP","via","composio","automate","first","Tools","search","tasks",{"skillNo":131,"name":132,"nameZh":6,"description":133,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":106,"licenseFileUrl":6,"sourceHtmlUrl":134,"categories":135,"keywords":137,"publishedAt":44},"01M6J7MK45K2ZZ0EWTP1N6SZ7F","zylvie-automation","Automate Zylvie tasks via Rube MCP (Composio). Always search tools first for current schemas.","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fzylvie-automation\u002FSKILL.md",[136],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},[119,138,122,129,29,123,124,125,126,127,128],"zylvie automation",{"skillNo":140,"name":141,"nameZh":6,"description":142,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":106,"licenseFileUrl":6,"sourceHtmlUrl":143,"categories":144,"keywords":146,"publishedAt":44},"01M6J7MGDCVZ3YCXGEAPR5V3MS","zoominfo-automation","Automate Zoominfo tasks via Rube MCP (Composio). Always search tools first for current schemas.","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fzoominfo-automation\u002FSKILL.md",[145],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},[147,148,119,125,124,127,123,129,29,128,122,126],"zoominfo","zoominfo automation",{"skillNo":150,"name":151,"nameZh":6,"description":152,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":106,"licenseFileUrl":6,"sourceHtmlUrl":153,"categories":154,"keywords":156,"publishedAt":44},"01M6J7MDK48N09B7C6H9TCPY9A","zoom-out","Tell the agent to zoom out and give broader context or a higher-level perspective. Use when you're unfamiliar with a section of code or need to understand how it fits into the bigger picture.","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fzoom-out\u002FSKILL.md",[155],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},[157,158,159,160,161,162,163,164,165,166,167,168,169],"zoom out","zoom","perspective","code","re","you","with","understand","it","use","context","how","true",{"skillNo":171,"name":172,"nameZh":6,"description":173,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":106,"licenseFileUrl":6,"sourceHtmlUrl":174,"categories":175,"keywords":177,"publishedAt":44},"01M6J7MAD47Q9A55AK3EXS914H","zoho-mail-automation","Automate Zoho Mail tasks via Rube MCP (Composio): email sending, folders, labels, and mailbox management. Always search tools first for current schemas.","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fzoho-mail-automation\u002FSKILL.md",[176],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},[178,179,180,181,119,182,128,129,122,127,183,29,123,184,124,185,125,126],"mail","zoho mail automation","zoho","zoho mail","email","labels","mailbox","management",{"skillNo":187,"name":188,"nameZh":6,"description":189,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":12,"repositoryNo":13,"starsCount":14,"licenseSpdxId":106,"licenseFileUrl":6,"sourceHtmlUrl":190,"categories":191,"keywords":196,"publishedAt":44},"01M6J7M76HTWPP8TRWWCDS42NB","zoho-invoice-automation","Automate Zoho Invoice tasks via Rube MCP (Composio): invoices, estimates, expenses, clients, and payment tracking. Always search tools first for current schemas.","https:\u002F\u002Fgithub.com\u002Fonfire7777\u002Funiversal-ai-skills-library\u002Fblob\u002Fmain\u002Fskills\u002Fzoho-invoice-automation\u002FSKILL.md",[192,193],{"categoryNo":19,"categoryName":20,"taxonomyType":21,"taxonomyTypeDesc":22},{"categoryNo":194,"categoryName":195,"taxonomyType":116,"taxonomyTypeDesc":117},"01JBSEEDCAT000000000000040","财务",[197,119,180,198,122,123,29,199,200,201,124,202,203,125,126,127,128,129],"invoice","zoho invoice automation","invoices","expenses","clients","payment","tracking"]