[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6FJVMCQJ2NP11GARVCB2MGJ":3,"skill-same:01M6FJVMCQJ2NP11GARVCB2MGJ":70},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":22,"publishedAt":37,"linkOnly":38,"markdownContent":39,"markdownBody":40,"frontMatter":41,"refName":43,"commitSha":44,"skillDirectory":45,"forksCount":46,"repositoryHtmlUrl":47,"repositoryHomepageUrl":6,"ownerHtmlUrl":48,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":49,"install":50,"resources":63},"01M6FJVMCQJ2NP11GARVCB2MGJ","document-classification-nlp",null,"Automatically classify and extract information from construction documents using NLP. Categorize RFIs, submittals, change orders, specifications, and contracts.","kbarbel640-del\u002Fskills","kbarbel640-del","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F255315490?v=4","01M6CJPPQGCCESXMG3K7BBGQ5J","01M6CJPPX2PB2BCZERFVM2C2B3",1,"MIT","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fdatadrivenconstruction\u002Fdocument-classification-nlp\u002FSKILL.md",[17],{"categoryNo":18,"categoryName":19,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000002","文档处理","SCENARIO","使用场景",[23,24,25,26,27,28,29,30,31,32,33,34,35,36],"nlp","document classification","classification","document","document classification nlp","change","documents","extract","categorize","contracts","orders","classify","information","specifications","2026-11-10 10:12:30",false,"---\r\nslug: \"document-classification-nlp\"\r\ndisplay_name: \"Document Classification Nlp\"\r\ndescription: \"Automatically classify and extract information from construction documents using NLP. Categorize RFIs, submittals, change orders, specifications, and contracts.\"\r\n---\r\n\r\n# Document Classification with NLP\r\n\r\n## Overview\r\n\r\nThis skill implements NLP-based document classification and information extraction for construction projects. Automate document sorting, key term extraction, and content analysis.\r\n\r\n**Document Types:**\r\n- RFIs (Requests for Information)\r\n- Submittals and shop drawings\r\n- Change orders and variations\r\n- Specifications and standards\r\n- Contracts and agreements\r\n- Safety reports and permits\r\n\r\n## Quick Start\r\n\r\n```python\r\nfrom sklearn.feature_extraction.text import TfidfVectorizer\r\nfrom sklearn.naive_bayes import MultinomialNB\r\nfrom sklearn.pipeline import Pipeline\r\nimport pandas as pd\r\n\r\n# Sample training data\r\ndocuments = [\r\n    (\"Please clarify the steel reinforcement spacing for the foundation slab\", \"RFI\"),\r\n    (\"Attached shop drawing for HVAC ductwork layout\", \"Submittal\"),\r\n    (\"Additional cost for unforeseen soil conditions\", \"Change Order\"),\r\n    (\"Fire-rated wall assembly specification Section 09 21 16\", \"Specification\"),\r\n]\r\n\r\ntexts, labels = zip(*documents)\r\n\r\n# Train classifier\r\nclassifier = Pipeline([\r\n    ('tfidf', TfidfVectorizer(max_features=1000, ngram_range=(1, 2))),\r\n    ('clf', MultinomialNB())\r\n])\r\n\r\nclassifier.fit(texts, labels)\r\n\r\n# Classify new document\r\nnew_doc = \"Request to approve substitution of specified light fixtures\"\r\nprediction = classifier.predict([new_doc])[0]\r\nprint(f\"Classification: {prediction}\")  # Output: Submittal\r\n```\r\n\r\n## Advanced Classification System\r\n\r\n### Document Classifier Class\r\n\r\n```python\r\nimport re\r\nimport pandas as pd\r\nimport numpy as np\r\nfrom sklearn.feature_extraction.text import TfidfVectorizer\r\nfrom sklearn.naive_bayes import MultinomialNB\r\nfrom sklearn.svm import LinearSVC\r\nfrom sklearn.ensemble import RandomForestClassifier\r\nfrom sklearn.pipeline import Pipeline\r\nfrom sklearn.model_selection import cross_val_score\r\nfrom sklearn.preprocessing import LabelEncoder\r\nfrom typing import List, Dict, Tuple, Optional\r\nimport spacy\r\nfrom dataclasses import dataclass\r\n\r\n@dataclass\r\nclass ClassificationResult:\r\n    document_id: str\r\n    predicted_class: str\r\n    confidence: float\r\n    alternative_classes: List[Tuple[str, float]]\r\n    extracted_entities: Dict[str, List[str]]\r\n    keywords: List[str]\r\n\r\nclass ConstructionDocumentClassifier:\r\n    \"\"\"Classify and analyze construction documents\"\"\"\r\n\r\n    # Document type patterns\r\n    DOCUMENT_PATTERNS = {\r\n        'RFI': [\r\n            r'request\\s+for\\s+information',\r\n            r'clarification\\s+(needed|required|requested)',\r\n            r'please\\s+(clarify|confirm|advise)',\r\n            r'question\\s+(regarding|about)',\r\n            r'rfi\\s*#?\\d*'\r\n        ],\r\n        'Submittal': [\r\n            r'submittal',\r\n            r'shop\\s+drawing',\r\n            r'product\\s+data',\r\n            r'sample\\s+submission',\r\n            r'approval\\s+request',\r\n            r'material\\s+submission'\r\n        ],\r\n        'Change Order': [\r\n            r'change\\s+order',\r\n            r'variation\\s+order',\r\n            r'cost\\s+(increase|adjustment|addition)',\r\n            r'scope\\s+change',\r\n            r'additional\\s+work',\r\n            r'unforeseen\\s+conditions'\r\n        ],\r\n        'Specification': [\r\n            r'section\\s+\\d{2}\\s+\\d{2}\\s+\\d{2}',\r\n            r'specification',\r\n            r'performance\\s+requirement',\r\n            r'material\\s+standard',\r\n            r'quality\\s+standard'\r\n        ],\r\n        'Safety Report': [\r\n            r'incident\\s+report',\r\n            r'safety\\s+(inspection|violation|observation)',\r\n            r'hazard\\s+(identification|assessment)',\r\n            r'near\\s+miss',\r\n            r'osha',\r\n            r'jha|jsa'\r\n        ],\r\n        'Contract': [\r\n            r'contract\\s+agreement',\r\n            r'terms\\s+and\\s+conditions',\r\n            r'scope\\s+of\\s+work',\r\n            r'payment\\s+terms',\r\n            r'warranty\\s+provision'\r\n        ]\r\n    }\r\n\r\n    def __init__(self, use_spacy: bool = True):\r\n        self.classifier = None\r\n        self.vectorizer = None\r\n        self.label_encoder = LabelEncoder()\r\n\r\n        if use_spacy:\r\n            try:\r\n                self.nlp = spacy.load(\"en_core_web_sm\")\r\n            except:\r\n                self.nlp = None\r\n        else:\r\n            self.nlp = None\r\n\r\n    def train(self, documents: List[str], labels: List[str]) -> Dict:\r\n        \"\"\"Train the document classifier\"\"\"\r\n        # Encode labels\r\n        y = self.label_encoder.fit_transform(labels)\r\n\r\n        # Create pipeline\r\n        self.classifier = Pipeline([\r\n            ('tfidf', TfidfVectorizer(\r\n                max_features=5000,\r\n                ngram_range=(1, 3),\r\n                stop_words='english',\r\n                sublinear_tf=True\r\n            )),\r\n            ('clf', LinearSVC(C=1.0, class_weight='balanced'))\r\n        ])\r\n\r\n        # Train\r\n        self.classifier.fit(documents, y)\r\n\r\n        # Cross-validation\r\n        scores = cross_val_score(self.classifier, documents, y, cv=5)\r\n\r\n        return {\r\n            'accuracy_mean': scores.mean(),\r\n            'accuracy_std': scores.std(),\r\n            'classes': list(self.label_encoder.classes_)\r\n        }\r\n\r\n    def classify(self, document: str) -> ClassificationResult:\r\n        \"\"\"Classify a single document\"\"\"\r\n        if self.classifier is None:\r\n            # Use rule-based classification if no model trained\r\n            return self._rule_based_classify(document)\r\n\r\n        # Get prediction\r\n        prediction = self.classifier.predict([document])[0]\r\n        predicted_class = self.label_encoder.inverse_transform([prediction])[0]\r\n\r\n        # Get confidence scores\r\n        decision_scores = self.classifier.decision_function([document])[0]\r\n        probs = self._softmax(decision_scores)\r\n\r\n        alternatives = [\r\n            (self.label_encoder.inverse_transform([i])[0], float(probs[i]))\r\n            for i in np.argsort(probs)[::-1][1:4]\r\n        ]\r\n\r\n        # Extract entities and keywords\r\n        entities = self._extract_entities(document)\r\n        keywords = self._extract_keywords(document)\r\n\r\n        return ClassificationResult(\r\n            document_id=\"\",\r\n            predicted_class=predicted_class,\r\n            confidence=float(probs[prediction]),\r\n            alternative_classes=alternatives,\r\n            extracted_entities=entities,\r\n            keywords=keywords\r\n        )\r\n\r\n    def _rule_based_classify(self, document: str) -> ClassificationResult:\r\n        \"\"\"Rule-based classification using patterns\"\"\"\r\n        doc_lower = document.lower()\r\n        scores = {}\r\n\r\n        for doc_type, patterns in self.DOCUMENT_PATTERNS.items():\r\n            score = sum(\r\n                1 for pattern in patterns\r\n                if re.search(pattern, doc_lower)\r\n            )\r\n            scores[doc_type] = score\r\n\r\n        if max(scores.values()) == 0:\r\n            predicted = 'Other'\r\n            confidence = 0.5\r\n        else:\r\n            predicted = max(scores, key=scores.get)\r\n            confidence = scores[predicted] \u002F len(self.DOCUMENT_PATTERNS[predicted])\r\n\r\n        return ClassificationResult(\r\n            document_id=\"\",\r\n            predicted_class=predicted,\r\n            confidence=confidence,\r\n            alternative_classes=[],\r\n            extracted_entities=self._extract_entities(document),\r\n            keywords=self._extract_keywords(document)\r\n        )\r\n\r\n    def _extract_entities(self, document: str) -> Dict[str, List[str]]:\r\n        \"\"\"Extract named entities from document\"\"\"\r\n        entities = {\r\n            'dates': [],\r\n            'organizations': [],\r\n            'people': [],\r\n            'monetary': [],\r\n            'references': []\r\n        }\r\n\r\n        # Date patterns\r\n        date_pattern = r'\\d{1,2}[\u002F-]\\d{1,2}[\u002F-]\\d{2,4}'\r\n        entities['dates'] = re.findall(date_pattern, document)\r\n\r\n        # Money patterns\r\n        money_pattern = r'\\$[\\d,]+(?:\\.\\d{2})?'\r\n        entities['monetary'] = re.findall(money_pattern, document)\r\n\r\n        # Reference numbers\r\n        ref_pattern = r'(?:RFI|CO|SI|PR)[-#]?\\s*\\d+'\r\n        entities['references'] = re.findall(ref_pattern, document, re.IGNORECASE)\r\n\r\n        # Use spaCy for NER if available\r\n        if self.nlp:\r\n            doc = self.nlp(document)\r\n            for ent in doc.ents:\r\n                if ent.label_ == 'ORG':\r\n                    entities['organizations'].append(ent.text)\r\n                elif ent.label_ == 'PERSON':\r\n                    entities['people'].append(ent.text)\r\n\r\n        return entities\r\n\r\n    def _extract_keywords(self, document: str, top_n: int = 10) -> List[str]:\r\n        \"\"\"Extract key terms from document\"\"\"\r\n        # Construction-specific terms\r\n        construction_terms = [\r\n            'concrete', 'steel', 'reinforcement', 'foundation', 'structural',\r\n            'hvac', 'plumbing', 'electrical', 'mechanical', 'architectural',\r\n            'specification', 'drawing', 'detail', 'schedule', 'submittals',\r\n            'rfi', 'change order', 'delay', 'inspection', 'approval'\r\n        ]\r\n\r\n        doc_lower = document.lower()\r\n        found_terms = [term for term in construction_terms if term in doc_lower]\r\n\r\n        return found_terms[:top_n]\r\n\r\n    def _softmax(self, x: np.ndarray) -> np.ndarray:\r\n        \"\"\"Convert decision scores to probabilities\"\"\"\r\n        exp_x = np.exp(x - np.max(x))\r\n        return exp_x \u002F exp_x.sum()\r\n\r\n    def batch_classify(self, documents: List[str]) -> pd.DataFrame:\r\n        \"\"\"Classify multiple documents\"\"\"\r\n        results = [self.classify(doc) for doc in documents]\r\n\r\n        return pd.DataFrame([{\r\n            'Predicted_Class': r.predicted_class,\r\n            'Confidence': r.confidence,\r\n            'Keywords': ', '.join(r.keywords),\r\n            'Dates_Found': ', '.join(r.extracted_entities['dates']),\r\n            'References_Found': ', '.join(r.extracted_entities['references'])\r\n        } for r in results])\r\n```\r\n\r\n## Information Extraction\r\n\r\n### Key Information Extractor\r\n\r\n```python\r\nclass ConstructionInfoExtractor:\r\n    \"\"\"Extract key information from construction documents\"\"\"\r\n\r\n    def __init__(self):\r\n        self.patterns = {\r\n            'rfi_number': r'RFI\\s*[-#]?\\s*(\\d+)',\r\n            'submittal_number': r'(?:Submittal|SI)\\s*[-#]?\\s*(\\d+)',\r\n            'change_order_number': r'(?:Change Order|CO|PCO)\\s*[-#]?\\s*(\\d+)',\r\n            'spec_section': r'Section\\s*(\\d{2}\\s*\\d{2}\\s*\\d{2})',\r\n            'cost_amount': r'\\$\\s*([\\d,]+(?:\\.\\d{2})?)',\r\n            'duration_days': r'(\\d+)\\s*(?:calendar\\s+)?days?',\r\n            'drawing_reference': r'(?:Drawing|Dwg|DWG)\\s*[-#]?\\s*([A-Z\\d-]+)',\r\n            'date': r'(\\d{1,2}[\u002F-]\\d{1,2}[\u002F-]\\d{2,4})',\r\n            'contractor_name': r'(?:Contractor|Subcontractor):\\s*([^\\n]+)',\r\n            'project_name': r'Project:\\s*([^\\n]+)',\r\n            'priority': r'(?:Priority|Urgency):\\s*(Critical|High|Medium|Low)'\r\n        }\r\n\r\n    def extract_all(self, document: str) -> Dict:\r\n        \"\"\"Extract all available information\"\"\"\r\n        results = {}\r\n\r\n        for field, pattern in self.patterns.items():\r\n            matches = re.findall(pattern, document, re.IGNORECASE)\r\n            results[field] = matches if matches else None\r\n\r\n        # Post-process\r\n        if results.get('cost_amount'):\r\n            results['cost_amount'] = [\r\n                float(amt.replace(',', ''))\r\n                for amt in results['cost_amount']\r\n            ]\r\n\r\n        return results\r\n\r\n    def extract_rfi_details(self, document: str) -> Dict:\r\n        \"\"\"Extract RFI-specific information\"\"\"\r\n        return {\r\n            'rfi_number': self._find_first(document, self.patterns['rfi_number']),\r\n            'date_submitted': self._find_first(document, self.patterns['date']),\r\n            'spec_section': self._find_first(document, self.patterns['spec_section']),\r\n            'drawing_ref': self._find_first(document, self.patterns['drawing_reference']),\r\n            'question': self._extract_question(document),\r\n            'priority': self._find_first(document, self.patterns['priority'])\r\n        }\r\n\r\n    def extract_change_order_details(self, document: str) -> Dict:\r\n        \"\"\"Extract change order specific information\"\"\"\r\n        costs = re.findall(self.patterns['cost_amount'], document)\r\n        total_cost = sum(float(c.replace(',', '')) for c in costs) if costs else None\r\n\r\n        return {\r\n            'co_number': self._find_first(document, self.patterns['change_order_number']),\r\n            'date': self._find_first(document, self.patterns['date']),\r\n            'cost_impact': total_cost,\r\n            'duration_impact': self._find_first(document, self.patterns['duration_days']),\r\n            'reason': self._extract_reason(document),\r\n            'contractor': self._find_first(document, self.patterns['contractor_name'])\r\n        }\r\n\r\n    def _find_first(self, document: str, pattern: str) -> Optional[str]:\r\n        match = re.search(pattern, document, re.IGNORECASE)\r\n        return match.group(1) if match else None\r\n\r\n    def _extract_question(self, document: str) -> Optional[str]:\r\n        \"\"\"Extract the question from an RFI\"\"\"\r\n        # Look for question markers\r\n        patterns = [\r\n            r'Question:\\s*(.+?)(?:\\n\\n|$)',\r\n            r'(?:Please\\s+)?(?:clarify|confirm|advise)(.+?)(?:\\.|$)',\r\n        ]\r\n        for pattern in patterns:\r\n            match = re.search(pattern, document, re.IGNORECASE | re.DOTALL)\r\n            if match:\r\n                return match.group(1).strip()[:500]\r\n        return None\r\n\r\n    def _extract_reason(self, document: str) -> Optional[str]:\r\n        \"\"\"Extract reason for change order\"\"\"\r\n        patterns = [\r\n            r'Reason:\\s*(.+?)(?:\\n\\n|$)',\r\n            r'(?:Due to|Because of)\\s*(.+?)(?:\\.|$)',\r\n        ]\r\n        for pattern in patterns:\r\n            match = re.search(pattern, document, re.IGNORECASE | re.DOTALL)\r\n            if match:\r\n                return match.group(1).strip()[:500]\r\n        return None\r\n```\r\n\r\n## Processing Pipeline\r\n\r\n```python\r\ndef process_document_batch(documents: List[str], output_path: str):\r\n    \"\"\"Process and classify a batch of documents\"\"\"\r\n    classifier = ConstructionDocumentClassifier()\r\n    extractor = ConstructionInfoExtractor()\r\n\r\n    results = []\r\n\r\n    for i, doc in enumerate(documents):\r\n        # Classify\r\n        classification = classifier.classify(doc)\r\n\r\n        # Extract info based on type\r\n        if classification.predicted_class == 'RFI':\r\n            extracted = extractor.extract_rfi_details(doc)\r\n        elif classification.predicted_class == 'Change Order':\r\n            extracted = extractor.extract_change_order_details(doc)\r\n        else:\r\n            extracted = extractor.extract_all(doc)\r\n\r\n        results.append({\r\n            'Document_ID': i + 1,\r\n            'Classification': classification.predicted_class,\r\n            'Confidence': classification.confidence,\r\n            'Keywords': ', '.join(classification.keywords),\r\n            **extracted\r\n        })\r\n\r\n    df = pd.DataFrame(results)\r\n    df.to_excel(output_path, index=False)\r\n\r\n    return df\r\n```\r\n\r\n## Quick Reference\r\n\r\n| Document Type | Key Patterns | Extracted Info |\r\n|--------------|--------------|----------------|\r\n| RFI | \"request for information\", \"clarify\" | Number, spec section, question |\r\n| Submittal | \"shop drawing\", \"approval request\" | Number, product, spec section |\r\n| Change Order | \"change order\", \"additional cost\" | Number, cost, duration impact |\r\n| Specification | \"Section XX XX XX\" | Section number, requirements |\r\n| Safety Report | \"incident\", \"hazard\" | Date, type, severity |\r\n\r\n## Resources\r\n\r\n- **spaCy**: https:\u002F\u002Fspacy.io\r\n- **Scikit-learn**: https:\u002F\u002Fscikit-learn.org\r\n- **DDC Website**: https:\u002F\u002Fdatadrivenconstruction.io\r\n\r\n## Next Steps\r\n\r\n- See `vector-search` for semantic document search\r\n- See `llm-data-automation` for advanced extraction\r\n- See `pdf-to-structured` for PDF processing\r\n","# Document Classification with NLP\r\n\r\n## Overview\r\n\r\nThis skill implements NLP-based document classification and information extraction for construction projects. Automate document sorting, key term extraction, and content analysis.\r\n\r\n**Document Types:**\r\n- RFIs (Requests for Information)\r\n- Submittals and shop drawings\r\n- Change orders and variations\r\n- Specifications and standards\r\n- Contracts and agreements\r\n- Safety reports and permits\r\n\r\n## Quick Start\r\n\r\n```python\r\nfrom sklearn.feature_extraction.text import TfidfVectorizer\r\nfrom sklearn.naive_bayes import MultinomialNB\r\nfrom sklearn.pipeline import Pipeline\r\nimport pandas as pd\r\n\r\n# Sample training data\r\ndocuments = [\r\n    (\"Please clarify the steel reinforcement spacing for the foundation slab\", \"RFI\"),\r\n    (\"Attached shop drawing for HVAC ductwork layout\", \"Submittal\"),\r\n    (\"Additional cost for unforeseen soil conditions\", \"Change Order\"),\r\n    (\"Fire-rated wall assembly specification Section 09 21 16\", \"Specification\"),\r\n]\r\n\r\ntexts, labels = zip(*documents)\r\n\r\n# Train classifier\r\nclassifier = Pipeline([\r\n    ('tfidf', TfidfVectorizer(max_features=1000, ngram_range=(1, 2))),\r\n    ('clf', MultinomialNB())\r\n])\r\n\r\nclassifier.fit(texts, labels)\r\n\r\n# Classify new document\r\nnew_doc = \"Request to approve substitution of specified light fixtures\"\r\nprediction = classifier.predict([new_doc])[0]\r\nprint(f\"Classification: {prediction}\")  # Output: Submittal\r\n```\r\n\r\n## Advanced Classification System\r\n\r\n### Document Classifier Class\r\n\r\n```python\r\nimport re\r\nimport pandas as pd\r\nimport numpy as np\r\nfrom sklearn.feature_extraction.text import TfidfVectorizer\r\nfrom sklearn.naive_bayes import MultinomialNB\r\nfrom sklearn.svm import LinearSVC\r\nfrom sklearn.ensemble import RandomForestClassifier\r\nfrom sklearn.pipeline import Pipeline\r\nfrom sklearn.model_selection import cross_val_score\r\nfrom sklearn.preprocessing import LabelEncoder\r\nfrom typing import List, Dict, Tuple, Optional\r\nimport spacy\r\nfrom dataclasses import dataclass\r\n\r\n@dataclass\r\nclass ClassificationResult:\r\n    document_id: str\r\n    predicted_class: str\r\n    confidence: float\r\n    alternative_classes: List[Tuple[str, float]]\r\n    extracted_entities: Dict[str, List[str]]\r\n    keywords: List[str]\r\n\r\nclass ConstructionDocumentClassifier:\r\n    \"\"\"Classify and analyze construction documents\"\"\"\r\n\r\n    # Document type patterns\r\n    DOCUMENT_PATTERNS = {\r\n        'RFI': [\r\n            r'request\\s+for\\s+information',\r\n            r'clarification\\s+(needed|required|requested)',\r\n            r'please\\s+(clarify|confirm|advise)',\r\n            r'question\\s+(regarding|about)',\r\n            r'rfi\\s*#?\\d*'\r\n        ],\r\n        'Submittal': [\r\n            r'submittal',\r\n            r'shop\\s+drawing',\r\n            r'product\\s+data',\r\n            r'sample\\s+submission',\r\n            r'approval\\s+request',\r\n            r'material\\s+submission'\r\n        ],\r\n        'Change Order': [\r\n            r'change\\s+order',\r\n            r'variation\\s+order',\r\n            r'cost\\s+(increase|adjustment|addition)',\r\n            r'scope\\s+change',\r\n            r'additional\\s+work',\r\n            r'unforeseen\\s+conditions'\r\n        ],\r\n        'Specification': [\r\n            r'section\\s+\\d{2}\\s+\\d{2}\\s+\\d{2}',\r\n            r'specification',\r\n            r'performance\\s+requirement',\r\n            r'material\\s+standard',\r\n            r'quality\\s+standard'\r\n        ],\r\n        'Safety Report': [\r\n            r'incident\\s+report',\r\n            r'safety\\s+(inspection|violation|observation)',\r\n            r'hazard\\s+(identification|assessment)',\r\n            r'near\\s+miss',\r\n            r'osha',\r\n            r'jha|jsa'\r\n        ],\r\n        'Contract': [\r\n            r'contract\\s+agreement',\r\n            r'terms\\s+and\\s+conditions',\r\n            r'scope\\s+of\\s+work',\r\n            r'payment\\s+terms',\r\n            r'warranty\\s+provision'\r\n        ]\r\n    }\r\n\r\n    def __init__(self, use_spacy: bool = True):\r\n        self.classifier = None\r\n        self.vectorizer = None\r\n        self.label_encoder = LabelEncoder()\r\n\r\n        if use_spacy:\r\n            try:\r\n                self.nlp = spacy.load(\"en_core_web_sm\")\r\n            except:\r\n                self.nlp = None\r\n        else:\r\n            self.nlp = None\r\n\r\n    def train(self, documents: List[str], labels: List[str]) -> Dict:\r\n        \"\"\"Train the document classifier\"\"\"\r\n        # Encode labels\r\n        y = self.label_encoder.fit_transform(labels)\r\n\r\n        # Create pipeline\r\n        self.classifier = Pipeline([\r\n            ('tfidf', TfidfVectorizer(\r\n                max_features=5000,\r\n                ngram_range=(1, 3),\r\n                stop_words='english',\r\n                sublinear_tf=True\r\n            )),\r\n            ('clf', LinearSVC(C=1.0, class_weight='balanced'))\r\n        ])\r\n\r\n        # Train\r\n        self.classifier.fit(documents, y)\r\n\r\n        # Cross-validation\r\n        scores = cross_val_score(self.classifier, documents, y, cv=5)\r\n\r\n        return {\r\n            'accuracy_mean': scores.mean(),\r\n            'accuracy_std': scores.std(),\r\n            'classes': list(self.label_encoder.classes_)\r\n        }\r\n\r\n    def classify(self, document: str) -> ClassificationResult:\r\n        \"\"\"Classify a single document\"\"\"\r\n        if self.classifier is None:\r\n            # Use rule-based classification if no model trained\r\n            return self._rule_based_classify(document)\r\n\r\n        # Get prediction\r\n        prediction = self.classifier.predict([document])[0]\r\n        predicted_class = self.label_encoder.inverse_transform([prediction])[0]\r\n\r\n        # Get confidence scores\r\n        decision_scores = self.classifier.decision_function([document])[0]\r\n        probs = self._softmax(decision_scores)\r\n\r\n        alternatives = [\r\n            (self.label_encoder.inverse_transform([i])[0], float(probs[i]))\r\n            for i in np.argsort(probs)[::-1][1:4]\r\n        ]\r\n\r\n        # Extract entities and keywords\r\n        entities = self._extract_entities(document)\r\n        keywords = self._extract_keywords(document)\r\n\r\n        return ClassificationResult(\r\n            document_id=\"\",\r\n            predicted_class=predicted_class,\r\n            confidence=float(probs[prediction]),\r\n            alternative_classes=alternatives,\r\n            extracted_entities=entities,\r\n            keywords=keywords\r\n        )\r\n\r\n    def _rule_based_classify(self, document: str) -> ClassificationResult:\r\n        \"\"\"Rule-based classification using patterns\"\"\"\r\n        doc_lower = document.lower()\r\n        scores = {}\r\n\r\n        for doc_type, patterns in self.DOCUMENT_PATTERNS.items():\r\n            score = sum(\r\n                1 for pattern in patterns\r\n                if re.search(pattern, doc_lower)\r\n            )\r\n            scores[doc_type] = score\r\n\r\n        if max(scores.values()) == 0:\r\n            predicted = 'Other'\r\n            confidence = 0.5\r\n        else:\r\n            predicted = max(scores, key=scores.get)\r\n            confidence = scores[predicted] \u002F len(self.DOCUMENT_PATTERNS[predicted])\r\n\r\n        return ClassificationResult(\r\n            document_id=\"\",\r\n            predicted_class=predicted,\r\n            confidence=confidence,\r\n            alternative_classes=[],\r\n            extracted_entities=self._extract_entities(document),\r\n            keywords=self._extract_keywords(document)\r\n        )\r\n\r\n    def _extract_entities(self, document: str) -> Dict[str, List[str]]:\r\n        \"\"\"Extract named entities from document\"\"\"\r\n        entities = {\r\n            'dates': [],\r\n            'organizations': [],\r\n            'people': [],\r\n            'monetary': [],\r\n            'references': []\r\n        }\r\n\r\n        # Date patterns\r\n        date_pattern = r'\\d{1,2}[\u002F-]\\d{1,2}[\u002F-]\\d{2,4}'\r\n        entities['dates'] = re.findall(date_pattern, document)\r\n\r\n        # Money patterns\r\n        money_pattern = r'\\$[\\d,]+(?:\\.\\d{2})?'\r\n        entities['monetary'] = re.findall(money_pattern, document)\r\n\r\n        # Reference numbers\r\n        ref_pattern = r'(?:RFI|CO|SI|PR)[-#]?\\s*\\d+'\r\n        entities['references'] = re.findall(ref_pattern, document, re.IGNORECASE)\r\n\r\n        # Use spaCy for NER if available\r\n        if self.nlp:\r\n            doc = self.nlp(document)\r\n            for ent in doc.ents:\r\n                if ent.label_ == 'ORG':\r\n                    entities['organizations'].append(ent.text)\r\n                elif ent.label_ == 'PERSON':\r\n                    entities['people'].append(ent.text)\r\n\r\n        return entities\r\n\r\n    def _extract_keywords(self, document: str, top_n: int = 10) -> List[str]:\r\n        \"\"\"Extract key terms from document\"\"\"\r\n        # Construction-specific terms\r\n        construction_terms = [\r\n            'concrete', 'steel', 'reinforcement', 'foundation', 'structural',\r\n            'hvac', 'plumbing', 'electrical', 'mechanical', 'architectural',\r\n            'specification', 'drawing', 'detail', 'schedule', 'submittals',\r\n            'rfi', 'change order', 'delay', 'inspection', 'approval'\r\n        ]\r\n\r\n        doc_lower = document.lower()\r\n        found_terms = [term for term in construction_terms if term in doc_lower]\r\n\r\n        return found_terms[:top_n]\r\n\r\n    def _softmax(self, x: np.ndarray) -> np.ndarray:\r\n        \"\"\"Convert decision scores to probabilities\"\"\"\r\n        exp_x = np.exp(x - np.max(x))\r\n        return exp_x \u002F exp_x.sum()\r\n\r\n    def batch_classify(self, documents: List[str]) -> pd.DataFrame:\r\n        \"\"\"Classify multiple documents\"\"\"\r\n        results = [self.classify(doc) for doc in documents]\r\n\r\n        return pd.DataFrame([{\r\n            'Predicted_Class': r.predicted_class,\r\n            'Confidence': r.confidence,\r\n            'Keywords': ', '.join(r.keywords),\r\n            'Dates_Found': ', '.join(r.extracted_entities['dates']),\r\n            'References_Found': ', '.join(r.extracted_entities['references'])\r\n        } for r in results])\r\n```\r\n\r\n## Information Extraction\r\n\r\n### Key Information Extractor\r\n\r\n```python\r\nclass ConstructionInfoExtractor:\r\n    \"\"\"Extract key information from construction documents\"\"\"\r\n\r\n    def __init__(self):\r\n        self.patterns = {\r\n            'rfi_number': r'RFI\\s*[-#]?\\s*(\\d+)',\r\n            'submittal_number': r'(?:Submittal|SI)\\s*[-#]?\\s*(\\d+)',\r\n            'change_order_number': r'(?:Change Order|CO|PCO)\\s*[-#]?\\s*(\\d+)',\r\n            'spec_section': r'Section\\s*(\\d{2}\\s*\\d{2}\\s*\\d{2})',\r\n            'cost_amount': r'\\$\\s*([\\d,]+(?:\\.\\d{2})?)',\r\n            'duration_days': r'(\\d+)\\s*(?:calendar\\s+)?days?',\r\n            'drawing_reference': r'(?:Drawing|Dwg|DWG)\\s*[-#]?\\s*([A-Z\\d-]+)',\r\n            'date': r'(\\d{1,2}[\u002F-]\\d{1,2}[\u002F-]\\d{2,4})',\r\n            'contractor_name': r'(?:Contractor|Subcontractor):\\s*([^\\n]+)',\r\n            'project_name': r'Project:\\s*([^\\n]+)',\r\n            'priority': r'(?:Priority|Urgency):\\s*(Critical|High|Medium|Low)'\r\n        }\r\n\r\n    def extract_all(self, document: str) -> Dict:\r\n        \"\"\"Extract all available information\"\"\"\r\n        results = {}\r\n\r\n        for field, pattern in self.patterns.items():\r\n            matches = re.findall(pattern, document, re.IGNORECASE)\r\n            results[field] = matches if matches else None\r\n\r\n        # Post-process\r\n        if results.get('cost_amount'):\r\n            results['cost_amount'] = [\r\n                float(amt.replace(',', ''))\r\n                for amt in results['cost_amount']\r\n            ]\r\n\r\n        return results\r\n\r\n    def extract_rfi_details(self, document: str) -> Dict:\r\n        \"\"\"Extract RFI-specific information\"\"\"\r\n        return {\r\n            'rfi_number': self._find_first(document, self.patterns['rfi_number']),\r\n            'date_submitted': self._find_first(document, self.patterns['date']),\r\n            'spec_section': self._find_first(document, self.patterns['spec_section']),\r\n            'drawing_ref': self._find_first(document, self.patterns['drawing_reference']),\r\n            'question': self._extract_question(document),\r\n            'priority': self._find_first(document, self.patterns['priority'])\r\n        }\r\n\r\n    def extract_change_order_details(self, document: str) -> Dict:\r\n        \"\"\"Extract change order specific information\"\"\"\r\n        costs = re.findall(self.patterns['cost_amount'], document)\r\n        total_cost = sum(float(c.replace(',', '')) for c in costs) if costs else None\r\n\r\n        return {\r\n            'co_number': self._find_first(document, self.patterns['change_order_number']),\r\n            'date': self._find_first(document, self.patterns['date']),\r\n            'cost_impact': total_cost,\r\n            'duration_impact': self._find_first(document, self.patterns['duration_days']),\r\n            'reason': self._extract_reason(document),\r\n            'contractor': self._find_first(document, self.patterns['contractor_name'])\r\n        }\r\n\r\n    def _find_first(self, document: str, pattern: str) -> Optional[str]:\r\n        match = re.search(pattern, document, re.IGNORECASE)\r\n        return match.group(1) if match else None\r\n\r\n    def _extract_question(self, document: str) -> Optional[str]:\r\n        \"\"\"Extract the question from an RFI\"\"\"\r\n        # Look for question markers\r\n        patterns = [\r\n            r'Question:\\s*(.+?)(?:\\n\\n|$)',\r\n            r'(?:Please\\s+)?(?:clarify|confirm|advise)(.+?)(?:\\.|$)',\r\n        ]\r\n        for pattern in patterns:\r\n            match = re.search(pattern, document, re.IGNORECASE | re.DOTALL)\r\n            if match:\r\n                return match.group(1).strip()[:500]\r\n        return None\r\n\r\n    def _extract_reason(self, document: str) -> Optional[str]:\r\n        \"\"\"Extract reason for change order\"\"\"\r\n        patterns = [\r\n            r'Reason:\\s*(.+?)(?:\\n\\n|$)',\r\n            r'(?:Due to|Because of)\\s*(.+?)(?:\\.|$)',\r\n        ]\r\n        for pattern in patterns:\r\n            match = re.search(pattern, document, re.IGNORECASE | re.DOTALL)\r\n            if match:\r\n                return match.group(1).strip()[:500]\r\n        return None\r\n```\r\n\r\n## Processing Pipeline\r\n\r\n```python\r\ndef process_document_batch(documents: List[str], output_path: str):\r\n    \"\"\"Process and classify a batch of documents\"\"\"\r\n    classifier = ConstructionDocumentClassifier()\r\n    extractor = ConstructionInfoExtractor()\r\n\r\n    results = []\r\n\r\n    for i, doc in enumerate(documents):\r\n        # Classify\r\n        classification = classifier.classify(doc)\r\n\r\n        # Extract info based on type\r\n        if classification.predicted_class == 'RFI':\r\n            extracted = extractor.extract_rfi_details(doc)\r\n        elif classification.predicted_class == 'Change Order':\r\n            extracted = extractor.extract_change_order_details(doc)\r\n        else:\r\n            extracted = extractor.extract_all(doc)\r\n\r\n        results.append({\r\n            'Document_ID': i + 1,\r\n            'Classification': classification.predicted_class,\r\n            'Confidence': classification.confidence,\r\n            'Keywords': ', '.join(classification.keywords),\r\n            **extracted\r\n        })\r\n\r\n    df = pd.DataFrame(results)\r\n    df.to_excel(output_path, index=False)\r\n\r\n    return df\r\n```\r\n\r\n## Quick Reference\r\n\r\n| Document Type | Key Patterns | Extracted Info |\r\n|--------------|--------------|----------------|\r\n| RFI | \"request for information\", \"clarify\" | Number, spec section, question |\r\n| Submittal | \"shop drawing\", \"approval request\" | Number, product, spec section |\r\n| Change Order | \"change order\", \"additional cost\" | Number, cost, duration impact |\r\n| Specification | \"Section XX XX XX\" | Section number, requirements |\r\n| Safety Report | \"incident\", \"hazard\" | Date, type, severity |\r\n\r\n## Resources\r\n\r\n- **spaCy**: https:\u002F\u002Fspacy.io\r\n- **Scikit-learn**: https:\u002F\u002Fscikit-learn.org\r\n- **DDC Website**: https:\u002F\u002Fdatadrivenconstruction.io\r\n\r\n## Next Steps\r\n\r\n- See `vector-search` for semantic document search\r\n- See `llm-data-automation` for advanced extraction\r\n- See `pdf-to-structured` for PDF processing\r\n",{"slug":5,"description":7,"display_name":42},"Document Classification Nlp","main","e7e336ac628c3e3bd771aff1aaa469c93778f94a","skills\u002Fdatadrivenconstruction\u002Fdocument-classification-nlp",0,"https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del","2026-08-23 22:39:37",{"source":8,"skillName":5,"commands":51,"prompt":61,"zipAvailable":62},[52,55,58],{"tool":53,"command":54},"npx","npx skills add kbarbel640-del\u002Fskills --skill document-classification-nlp",{"tool":56,"command":57},"bunx","bunx skills add kbarbel640-del\u002Fskills --skill document-classification-nlp",{"tool":59,"command":60},"pnpm","pnpm dlx skills add kbarbel640-del\u002Fskills --skill document-classification-nlp","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fdatadrivenconstruction\u002Fdocument-classification-nlp\u002FSKILL.md",true,[64],{"path":65,"resourceType":66,"resourceTypeDesc":67,"sizeBytes":68,"sourceHtmlUrl":69},"_meta.json","REFERENCE","参考文档",322,"https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fdatadrivenconstruction\u002Fdocument-classification-nlp\u002F_meta.json",[71,102,129,150,156,176],{"skillNo":72,"name":73,"nameZh":6,"description":74,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":75,"categories":76,"keywords":82,"publishedAt":37},"01M6G8J55967D6281ACMKAKTEQ","crowd-prompting","A marketplace where AI agents improve prompts, system instructions, tool descriptions, and other text-based content with domain expertise from real-world operations — and earn tokens for valuable contributions.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzzadrian\u002Fcrowd-prompting\u002FSKILL.md",[77],{"categoryNo":78,"categoryName":79,"taxonomyType":80,"taxonomyTypeDesc":81},"01JBSEEDCAT000000000000036","内容创作者","PROFESSION","适用职业",[83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101],"crowd prompting","prompting","api","with","tokens","domain","world","prompts","operations","instructions","text","tool","system","improve","productivity","marketplace","https","content","ai agents",{"skillNo":103,"name":104,"nameZh":6,"description":105,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":106,"categories":107,"keywords":111,"publishedAt":37},"01M6G8J0PKPAYVET6BWE8W4C26","dilemm-ai","Compete in DilemmAI, the prisoner's dilemma AI arena at dilemm.ai. Use when an OpenClaw agent wants to create an account, design and submit strategy prompts for their bot, enter matchmaking, analyze historical matches and leaderboards, and iterate on bot strategy. Requires headless browser automation (no API available yet).","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzyori\u002Fdilemm-ai\u002FSKILL.md",[108],{"categoryNo":109,"categoryName":110,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000023","Skill与智能体开发",[112,85,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,90,128],"dilemm ai","arena","analyze","browser","openclaw agent","create","automation","browser automation","bot","openclaw","design","strategy","headless","compete","submit","matchmaking","use",{"skillNo":130,"name":131,"nameZh":6,"description":132,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":133,"categories":134,"keywords":138,"publishedAt":37},"01M6G8HWAF3K9SW7NP3Z83CPJ6","arxiv-paper-reviews","Interact with arXiv Crawler API to fetch papers, read reviews, and submit comments. Use when working with arXiv papers, fetching paper lists by date\u002Fcategory\u002Finterest, viewing paper details with comments, or submitting paper reviews via API at http:\u002F\u002F150.158.152.82:8000.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzxrys\u002Fweak-accept\u002FSKILL.md",[135],{"categoryNo":136,"categoryName":137,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000021","资料检索与研究",[139,140,141,142,143,144,145,86,146,126,128,147,85,148,149],"arxiv paper reviews","reviews","arxiv","paper","crawler","papers","comments","fetch","read","http","via",{"skillNo":151,"name":131,"nameZh":6,"description":132,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":152,"categories":153,"keywords":155,"publishedAt":37},"01M6G8HQJ7N7WMY6YV7D7SSJ16","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzxrys\u002Farxiv-paper-reviews\u002FSKILL.md",[154],{"categoryNo":136,"categoryName":137,"taxonomyType":20,"taxonomyTypeDesc":21},[142,141,140,139,149,144,145,86,143,126,128,146,147,85,148],{"skillNo":157,"name":158,"nameZh":6,"description":159,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":160,"categories":161,"keywords":162,"publishedAt":37},"01M6G8HM95Y64JNVFC2HN4E2BG","git-summary","Get a quick summary of the current Git repository including status, recent commits, branches, and contributors.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzweack\u002Fgit-summary\u002FSKILL.md",[],[163,164,165,166,167,168,169,170,171,172,173,174,175],"git","summary","git summary","commits","repository","win32","status","linux","contributors","true","quick","get","branches",{"skillNo":177,"name":178,"nameZh":6,"description":179,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":180,"categories":181,"keywords":186,"publishedAt":37},"01M6G8HGEGF05CNEJCGVYFSC4R","agent-constitution","Interact with AgentConstitution governance contracts on Base Sepolia. Check compliance, read rules, log actions, query governance state.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fztsalexey\u002Fagent-constitution\u002FSKILL.md",[182,183],{"categoryNo":109,"categoryName":110,"taxonomyType":20,"taxonomyTypeDesc":21},{"categoryNo":184,"categoryName":185,"taxonomyType":80,"taxonomyTypeDesc":81},"01JBSEEDCAT000000000000037","律师与法务",[187,188,32,189,86,190,191,43,192,147,193,194,195,196,197,99,198,199,200],"agent constitution","constitution","tree","query","base","log","check","Rules","projects","compliance","actions","state","governance","github"]