[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"skill:01M6FJP8FW8TTDG6SN5F253RJC":3,"skill-same:01M6FJP8FW8TTDG6SN5F253RJC":65},{"skillNo":4,"name":5,"nameZh":6,"description":7,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":15,"categories":16,"keywords":22,"publishedAt":32,"linkOnly":33,"markdownContent":34,"markdownBody":35,"frontMatter":36,"refName":38,"commitSha":39,"skillDirectory":40,"forksCount":41,"repositoryHtmlUrl":42,"repositoryHomepageUrl":6,"ownerHtmlUrl":43,"ownerBlogUrl":6,"ownerDescription":6,"updatedAt":44,"install":45,"resources":58},"01M6FJP8FW8TTDG6SN5F253RJC","cwicr-data-validator",null,"Validate CWICR data quality and estimate inputs. Check for errors, inconsistencies, outliers, and missing data.","kbarbel640-del\u002Fskills","kbarbel640-del","https:\u002F\u002Favatars.githubusercontent.com\u002Fu\u002F255315490?v=4","01M6CJPPQGCCESXMG3K7BBGQ5J","01M6CJPPX2PB2BCZERFVM2C2B3",1,"MIT","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fdatadrivenconstruction\u002Fcwicr-data-validator\u002FSKILL.md",[17],{"categoryNo":18,"categoryName":19,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000033","数据分析师","PROFESSION","适用职业",[23,24,25,26,27,28,29,30,31],"data","data validator","cwicr data validator","validate","data quality","quality","check","missing","errors","2026-11-10 10:12:30",false,"---\r\nslug: \"cwicr-data-validator\"\r\ndisplay_name: \"CWICR Data Validator\"\r\ndescription: \"Validate CWICR data quality and estimate inputs. Check for errors, inconsistencies, outliers, and missing data.\"\r\n---\r\n\r\n# CWICR Data Validator\r\n\r\n## Business Case\r\n\r\n### Problem Statement\r\nData quality issues cause:\r\n- Incorrect estimates\r\n- Budget overruns\r\n- Delayed projects\r\n- Rework costs\r\n\r\n### Solution\r\nSystematic validation of CWICR data and estimate inputs to catch errors, outliers, and inconsistencies before they impact projects.\r\n\r\n### Business Value\r\n- **Error prevention** - Catch issues early\r\n- **Data quality** - Ensure reliable estimates\r\n- **Consistency** - Standard validation rules\r\n- **Audit trail** - Document data issues\r\n\r\n## Technical Implementation\r\n\r\n```python\r\nimport pandas as pd\r\nimport numpy as np\r\nfrom typing import Dict, Any, List, Optional, Tuple\r\nfrom dataclasses import dataclass, field\r\nfrom enum import Enum\r\nfrom datetime import datetime\r\n\r\n\r\nclass ValidationSeverity(Enum):\r\n    \"\"\"Validation issue severity.\"\"\"\r\n    ERROR = \"error\"          # Must fix\r\n    WARNING = \"warning\"      # Should review\r\n    INFO = \"info\"            # For awareness\r\n\r\n\r\nclass ValidationCategory(Enum):\r\n    \"\"\"Validation categories.\"\"\"\r\n    MISSING_DATA = \"missing_data\"\r\n    INVALID_VALUE = \"invalid_value\"\r\n    OUTLIER = \"outlier\"\r\n    DUPLICATE = \"duplicate\"\r\n    INCONSISTENT = \"inconsistent\"\r\n    FORMAT = \"format\"\r\n\r\n\r\n@dataclass\r\nclass ValidationIssue:\r\n    \"\"\"Single validation issue.\"\"\"\r\n    field: str\r\n    record_id: str\r\n    category: ValidationCategory\r\n    severity: ValidationSeverity\r\n    message: str\r\n    current_value: Any\r\n    expected: str\r\n\r\n\r\n@dataclass\r\nclass ValidationResult:\r\n    \"\"\"Complete validation result.\"\"\"\r\n    total_records: int\r\n    valid_records: int\r\n    issues: List[ValidationIssue]\r\n    error_count: int\r\n    warning_count: int\r\n    info_count: int\r\n    validation_date: datetime\r\n    passed: bool\r\n\r\n\r\nclass CWICRDataValidator:\r\n    \"\"\"Validate CWICR data and estimates.\"\"\"\r\n\r\n    # Standard validation rules\r\n    REQUIRED_FIELDS = ['work_item_code', 'description', 'unit']\r\n    NUMERIC_FIELDS = ['labor_cost', 'material_cost', 'equipment_cost', 'labor_norm']\r\n    POSITIVE_FIELDS = ['labor_cost', 'material_cost', 'equipment_cost', 'quantity']\r\n\r\n    # Outlier detection thresholds (IQR multiplier)\r\n    OUTLIER_THRESHOLD = 3.0\r\n\r\n    def __init__(self, cwicr_reference: pd.DataFrame = None):\r\n        self.reference = cwicr_reference\r\n        if cwicr_reference is not None:\r\n            self._build_reference_stats()\r\n\r\n    def _build_reference_stats(self):\r\n        \"\"\"Build reference statistics for outlier detection.\"\"\"\r\n        self._stats = {}\r\n\r\n        for col in self.NUMERIC_FIELDS:\r\n            if col in self.reference.columns:\r\n                values = pd.to_numeric(self.reference[col], errors='coerce').dropna()\r\n                if len(values) > 0:\r\n                    self._stats[col] = {\r\n                        'mean': values.mean(),\r\n                        'std': values.std(),\r\n                        'q1': values.quantile(0.25),\r\n                        'q3': values.quantile(0.75),\r\n                        'iqr': values.quantile(0.75) - values.quantile(0.25)\r\n                    }\r\n\r\n    def validate_dataframe(self, df: pd.DataFrame) -> ValidationResult:\r\n        \"\"\"Validate entire dataframe.\"\"\"\r\n\r\n        issues = []\r\n        valid_count = 0\r\n\r\n        for idx, row in df.iterrows():\r\n            row_issues = self._validate_row(row, str(idx))\r\n            issues.extend(row_issues)\r\n\r\n            if not any(i.severity == ValidationSeverity.ERROR for i in row_issues):\r\n                valid_count += 1\r\n\r\n        # Check for duplicates\r\n        if 'work_item_code' in df.columns:\r\n            duplicates = df[df.duplicated(subset=['work_item_code'], keep=False)]\r\n            for idx, row in duplicates.iterrows():\r\n                issues.append(ValidationIssue(\r\n                    field='work_item_code',\r\n                    record_id=str(idx),\r\n                    category=ValidationCategory.DUPLICATE,\r\n                    severity=ValidationSeverity.WARNING,\r\n                    message=f\"Duplicate work item code: {row['work_item_code']}\",\r\n                    current_value=row['work_item_code'],\r\n                    expected=\"Unique codes\"\r\n                ))\r\n\r\n        error_count = sum(1 for i in issues if i.severity == ValidationSeverity.ERROR)\r\n        warning_count = sum(1 for i in issues if i.severity == ValidationSeverity.WARNING)\r\n        info_count = sum(1 for i in issues if i.severity == ValidationSeverity.INFO)\r\n\r\n        return ValidationResult(\r\n            total_records=len(df),\r\n            valid_records=valid_count,\r\n            issues=issues,\r\n            error_count=error_count,\r\n            warning_count=warning_count,\r\n            info_count=info_count,\r\n            validation_date=datetime.now(),\r\n            passed=error_count == 0\r\n        )\r\n\r\n    def _validate_row(self, row: pd.Series, record_id: str) -> List[ValidationIssue]:\r\n        \"\"\"Validate single row.\"\"\"\r\n\r\n        issues = []\r\n\r\n        # Check required fields\r\n        for field in self.REQUIRED_FIELDS:\r\n            if field in row.index:\r\n                value = row[field]\r\n                if pd.isna(value) or str(value).strip() == '':\r\n                    issues.append(ValidationIssue(\r\n                        field=field,\r\n                        record_id=record_id,\r\n                        category=ValidationCategory.MISSING_DATA,\r\n                        severity=ValidationSeverity.ERROR,\r\n                        message=f\"Required field '{field}' is missing\",\r\n                        current_value=value,\r\n                        expected=\"Non-empty value\"\r\n                    ))\r\n\r\n        # Check numeric fields\r\n        for field in self.NUMERIC_FIELDS:\r\n            if field in row.index:\r\n                value = row[field]\r\n                if pd.notna(value):\r\n                    try:\r\n                        num_val = float(value)\r\n                        # Check for negative where positive expected\r\n                        if field in self.POSITIVE_FIELDS and num_val \u003C 0:\r\n                            issues.append(ValidationIssue(\r\n                                field=field,\r\n                                record_id=record_id,\r\n                                category=ValidationCategory.INVALID_VALUE,\r\n                                severity=ValidationSeverity.ERROR,\r\n                                message=f\"Negative value in '{field}'\",\r\n                                current_value=value,\r\n                                expected=\"Positive number\"\r\n                            ))\r\n\r\n                        # Check for outliers\r\n                        if self._stats and field in self._stats:\r\n                            stats = self._stats[field]\r\n                            lower = stats['q1'] - self.OUTLIER_THRESHOLD * stats['iqr']\r\n                            upper = stats['q3'] + self.OUTLIER_THRESHOLD * stats['iqr']\r\n\r\n                            if num_val \u003C lower or num_val > upper:\r\n                                issues.append(ValidationIssue(\r\n                                    field=field,\r\n                                    record_id=record_id,\r\n                                    category=ValidationCategory.OUTLIER,\r\n                                    severity=ValidationSeverity.WARNING,\r\n                                    message=f\"Outlier value in '{field}'\",\r\n                                    current_value=value,\r\n                                    expected=f\"Between {lower:.2f} and {upper:.2f}\"\r\n                                ))\r\n\r\n                    except (ValueError, TypeError):\r\n                        issues.append(ValidationIssue(\r\n                            field=field,\r\n                            record_id=record_id,\r\n                            category=ValidationCategory.INVALID_VALUE,\r\n                            severity=ValidationSeverity.ERROR,\r\n                            message=f\"Non-numeric value in '{field}'\",\r\n                            current_value=value,\r\n                            expected=\"Numeric value\"\r\n                        ))\r\n\r\n        # Check work item code format\r\n        if 'work_item_code' in row.index:\r\n            code = row['work_item_code']\r\n            if pd.notna(code) and not self._valid_code_format(str(code)):\r\n                issues.append(ValidationIssue(\r\n                    field='work_item_code',\r\n                    record_id=record_id,\r\n                    category=ValidationCategory.FORMAT,\r\n                    severity=ValidationSeverity.INFO,\r\n                    message=\"Non-standard code format\",\r\n                    current_value=code,\r\n                    expected=\"CATEGORY-NUMBER format\"\r\n                ))\r\n\r\n        return issues\r\n\r\n    def _valid_code_format(self, code: str) -> bool:\r\n        \"\"\"Check if code follows expected format.\"\"\"\r\n        # Expect format like \"CONC-001\" or \"EXCV-DEEP-002\"\r\n        parts = code.split('-')\r\n        return len(parts) >= 2 and parts[0].isalpha()\r\n\r\n    def validate_estimate(self,\r\n                          items: List[Dict[str, Any]],\r\n                          check_against_cwicr: bool = True) -> ValidationResult:\r\n        \"\"\"Validate estimate items.\"\"\"\r\n\r\n        issues = []\r\n        valid_count = 0\r\n\r\n        for i, item in enumerate(items):\r\n            record_id = str(i)\r\n            item_issues = []\r\n\r\n            # Check required fields\r\n            code = item.get('work_item_code', item.get('code'))\r\n            if not code:\r\n                item_issues.append(ValidationIssue(\r\n                    field='work_item_code',\r\n                    record_id=record_id,\r\n                    category=ValidationCategory.MISSING_DATA,\r\n                    severity=ValidationSeverity.ERROR,\r\n                    message=\"Missing work item code\",\r\n                    current_value=None,\r\n                    expected=\"Valid work item code\"\r\n                ))\r\n\r\n            # Check quantity\r\n            qty = item.get('quantity', 0)\r\n            if qty \u003C= 0:\r\n                item_issues.append(ValidationIssue(\r\n                    field='quantity',\r\n                    record_id=record_id,\r\n                    category=ValidationCategory.INVALID_VALUE,\r\n                    severity=ValidationSeverity.ERROR,\r\n                    message=\"Invalid quantity\",\r\n                    current_value=qty,\r\n                    expected=\"Positive number\"\r\n                ))\r\n\r\n            # Check against CWICR reference\r\n            if check_against_cwicr and self.reference is not None and code:\r\n                if 'work_item_code' in self.reference.columns:\r\n                    if code not in self.reference['work_item_code'].values:\r\n                        item_issues.append(ValidationIssue(\r\n                            field='work_item_code',\r\n                            record_id=record_id,\r\n                            category=ValidationCategory.INVALID_VALUE,\r\n                            severity=ValidationSeverity.WARNING,\r\n                            message=f\"Work item code not found in CWICR: {code}\",\r\n                            current_value=code,\r\n                            expected=\"Valid CWICR code\"\r\n                        ))\r\n\r\n            issues.extend(item_issues)\r\n\r\n            if not any(i.severity == ValidationSeverity.ERROR for i in item_issues):\r\n                valid_count += 1\r\n\r\n        return ValidationResult(\r\n            total_records=len(items),\r\n            valid_records=valid_count,\r\n            issues=issues,\r\n            error_count=sum(1 for i in issues if i.severity == ValidationSeverity.ERROR),\r\n            warning_count=sum(1 for i in issues if i.severity == ValidationSeverity.WARNING),\r\n            info_count=sum(1 for i in issues if i.severity == ValidationSeverity.INFO),\r\n            validation_date=datetime.now(),\r\n            passed=all(i.severity != ValidationSeverity.ERROR for i in issues)\r\n        )\r\n\r\n    def get_data_quality_score(self, result: ValidationResult) -> Dict[str, Any]:\r\n        \"\"\"Calculate data quality score.\"\"\"\r\n\r\n        if result.total_records == 0:\r\n            return {'score': 0, 'grade': 'N\u002FA'}\r\n\r\n        # Weighted scoring\r\n        error_weight = 10\r\n        warning_weight = 3\r\n        info_weight = 1\r\n\r\n        total_deductions = (\r\n            result.error_count * error_weight +\r\n            result.warning_count * warning_weight +\r\n            result.info_count * info_weight\r\n        )\r\n\r\n        max_deductions = result.total_records * error_weight\r\n        score = max(0, 100 - (total_deductions \u002F max_deductions * 100)) if max_deductions > 0 else 100\r\n\r\n        # Assign grade\r\n        if score >= 95:\r\n            grade = 'A'\r\n        elif score >= 85:\r\n            grade = 'B'\r\n        elif score >= 75:\r\n            grade = 'C'\r\n        elif score >= 60:\r\n            grade = 'D'\r\n        else:\r\n            grade = 'F'\r\n\r\n        return {\r\n            'score': round(score, 1),\r\n            'grade': grade,\r\n            'total_records': result.total_records,\r\n            'valid_records': result.valid_records,\r\n            'error_count': result.error_count,\r\n            'warning_count': result.warning_count\r\n        }\r\n\r\n    def export_validation_report(self,\r\n                                  result: ValidationResult,\r\n                                  output_path: str) -> str:\r\n        \"\"\"Export validation report to Excel.\"\"\"\r\n\r\n        with pd.ExcelWriter(output_path, engine='openpyxl') as writer:\r\n            # Summary\r\n            quality = self.get_data_quality_score(result)\r\n            summary_df = pd.DataFrame([{\r\n                'Total Records': result.total_records,\r\n                'Valid Records': result.valid_records,\r\n                'Errors': result.error_count,\r\n                'Warnings': result.warning_count,\r\n                'Info': result.info_count,\r\n                'Quality Score': quality['score'],\r\n                'Grade': quality['grade'],\r\n                'Validation Date': result.validation_date,\r\n                'Passed': result.passed\r\n            }])\r\n            summary_df.to_excel(writer, sheet_name='Summary', index=False)\r\n\r\n            # Issues\r\n            if result.issues:\r\n                issues_df = pd.DataFrame([\r\n                    {\r\n                        'Record': i.record_id,\r\n                        'Field': i.field,\r\n                        'Category': i.category.value,\r\n                        'Severity': i.severity.value,\r\n                        'Message': i.message,\r\n                        'Current Value': str(i.current_value),\r\n                        'Expected': i.expected\r\n                    }\r\n                    for i in result.issues\r\n                ])\r\n                issues_df.to_excel(writer, sheet_name='Issues', index=False)\r\n\r\n        return output_path\r\n```\r\n\r\n## Quick Start\r\n\r\n```python\r\n# Load CWICR reference\r\ncwicr = pd.read_parquet(\"ddc_cwicr_en.parquet\")\r\n\r\n# Initialize validator\r\nvalidator = CWICRDataValidator(cwicr)\r\n\r\n# Validate estimate\r\nitems = [\r\n    {'work_item_code': 'CONC-001', 'quantity': 100},\r\n    {'work_item_code': 'INVALID-CODE', 'quantity': -5}\r\n]\r\n\r\nresult = validator.validate_estimate(items)\r\nprint(f\"Passed: {result.passed}\")\r\nprint(f\"Errors: {result.error_count}\")\r\n```\r\n\r\n## Common Use Cases\r\n\r\n### 1. Data Quality Score\r\n```python\r\nquality = validator.get_data_quality_score(result)\r\nprint(f\"Score: {quality['score']} ({quality['grade']})\")\r\n```\r\n\r\n### 2. Validate DataFrame\r\n```python\r\nimport_df = pd.read_excel(\"estimate_import.xlsx\")\r\nresult = validator.validate_dataframe(import_df)\r\n\r\nfor issue in result.issues:\r\n    if issue.severity == ValidationSeverity.ERROR:\r\n        print(f\"ERROR: {issue.message}\")\r\n```\r\n\r\n### 3. Export Report\r\n```python\r\nvalidator.export_validation_report(result, \"validation_report.xlsx\")\r\n```\r\n\r\n## Resources\r\n- **GitHub**: [OpenConstructionEstimate-DDC-CWICR](https:\u002F\u002Fgithub.com\u002Fdatadrivenconstruction\u002FOpenConstructionEstimate-DDC-CWICR)\r\n- **DDC Book**: Chapter 2.1 - Data Quality Management\r\n","# CWICR Data Validator\r\n\r\n## Business Case\r\n\r\n### Problem Statement\r\nData quality issues cause:\r\n- Incorrect estimates\r\n- Budget overruns\r\n- Delayed projects\r\n- Rework costs\r\n\r\n### Solution\r\nSystematic validation of CWICR data and estimate inputs to catch errors, outliers, and inconsistencies before they impact projects.\r\n\r\n### Business Value\r\n- **Error prevention** - Catch issues early\r\n- **Data quality** - Ensure reliable estimates\r\n- **Consistency** - Standard validation rules\r\n- **Audit trail** - Document data issues\r\n\r\n## Technical Implementation\r\n\r\n```python\r\nimport pandas as pd\r\nimport numpy as np\r\nfrom typing import Dict, Any, List, Optional, Tuple\r\nfrom dataclasses import dataclass, field\r\nfrom enum import Enum\r\nfrom datetime import datetime\r\n\r\n\r\nclass ValidationSeverity(Enum):\r\n    \"\"\"Validation issue severity.\"\"\"\r\n    ERROR = \"error\"          # Must fix\r\n    WARNING = \"warning\"      # Should review\r\n    INFO = \"info\"            # For awareness\r\n\r\n\r\nclass ValidationCategory(Enum):\r\n    \"\"\"Validation categories.\"\"\"\r\n    MISSING_DATA = \"missing_data\"\r\n    INVALID_VALUE = \"invalid_value\"\r\n    OUTLIER = \"outlier\"\r\n    DUPLICATE = \"duplicate\"\r\n    INCONSISTENT = \"inconsistent\"\r\n    FORMAT = \"format\"\r\n\r\n\r\n@dataclass\r\nclass ValidationIssue:\r\n    \"\"\"Single validation issue.\"\"\"\r\n    field: str\r\n    record_id: str\r\n    category: ValidationCategory\r\n    severity: ValidationSeverity\r\n    message: str\r\n    current_value: Any\r\n    expected: str\r\n\r\n\r\n@dataclass\r\nclass ValidationResult:\r\n    \"\"\"Complete validation result.\"\"\"\r\n    total_records: int\r\n    valid_records: int\r\n    issues: List[ValidationIssue]\r\n    error_count: int\r\n    warning_count: int\r\n    info_count: int\r\n    validation_date: datetime\r\n    passed: bool\r\n\r\n\r\nclass CWICRDataValidator:\r\n    \"\"\"Validate CWICR data and estimates.\"\"\"\r\n\r\n    # Standard validation rules\r\n    REQUIRED_FIELDS = ['work_item_code', 'description', 'unit']\r\n    NUMERIC_FIELDS = ['labor_cost', 'material_cost', 'equipment_cost', 'labor_norm']\r\n    POSITIVE_FIELDS = ['labor_cost', 'material_cost', 'equipment_cost', 'quantity']\r\n\r\n    # Outlier detection thresholds (IQR multiplier)\r\n    OUTLIER_THRESHOLD = 3.0\r\n\r\n    def __init__(self, cwicr_reference: pd.DataFrame = None):\r\n        self.reference = cwicr_reference\r\n        if cwicr_reference is not None:\r\n            self._build_reference_stats()\r\n\r\n    def _build_reference_stats(self):\r\n        \"\"\"Build reference statistics for outlier detection.\"\"\"\r\n        self._stats = {}\r\n\r\n        for col in self.NUMERIC_FIELDS:\r\n            if col in self.reference.columns:\r\n                values = pd.to_numeric(self.reference[col], errors='coerce').dropna()\r\n                if len(values) > 0:\r\n                    self._stats[col] = {\r\n                        'mean': values.mean(),\r\n                        'std': values.std(),\r\n                        'q1': values.quantile(0.25),\r\n                        'q3': values.quantile(0.75),\r\n                        'iqr': values.quantile(0.75) - values.quantile(0.25)\r\n                    }\r\n\r\n    def validate_dataframe(self, df: pd.DataFrame) -> ValidationResult:\r\n        \"\"\"Validate entire dataframe.\"\"\"\r\n\r\n        issues = []\r\n        valid_count = 0\r\n\r\n        for idx, row in df.iterrows():\r\n            row_issues = self._validate_row(row, str(idx))\r\n            issues.extend(row_issues)\r\n\r\n            if not any(i.severity == ValidationSeverity.ERROR for i in row_issues):\r\n                valid_count += 1\r\n\r\n        # Check for duplicates\r\n        if 'work_item_code' in df.columns:\r\n            duplicates = df[df.duplicated(subset=['work_item_code'], keep=False)]\r\n            for idx, row in duplicates.iterrows():\r\n                issues.append(ValidationIssue(\r\n                    field='work_item_code',\r\n                    record_id=str(idx),\r\n                    category=ValidationCategory.DUPLICATE,\r\n                    severity=ValidationSeverity.WARNING,\r\n                    message=f\"Duplicate work item code: {row['work_item_code']}\",\r\n                    current_value=row['work_item_code'],\r\n                    expected=\"Unique codes\"\r\n                ))\r\n\r\n        error_count = sum(1 for i in issues if i.severity == ValidationSeverity.ERROR)\r\n        warning_count = sum(1 for i in issues if i.severity == ValidationSeverity.WARNING)\r\n        info_count = sum(1 for i in issues if i.severity == ValidationSeverity.INFO)\r\n\r\n        return ValidationResult(\r\n            total_records=len(df),\r\n            valid_records=valid_count,\r\n            issues=issues,\r\n            error_count=error_count,\r\n            warning_count=warning_count,\r\n            info_count=info_count,\r\n            validation_date=datetime.now(),\r\n            passed=error_count == 0\r\n        )\r\n\r\n    def _validate_row(self, row: pd.Series, record_id: str) -> List[ValidationIssue]:\r\n        \"\"\"Validate single row.\"\"\"\r\n\r\n        issues = []\r\n\r\n        # Check required fields\r\n        for field in self.REQUIRED_FIELDS:\r\n            if field in row.index:\r\n                value = row[field]\r\n                if pd.isna(value) or str(value).strip() == '':\r\n                    issues.append(ValidationIssue(\r\n                        field=field,\r\n                        record_id=record_id,\r\n                        category=ValidationCategory.MISSING_DATA,\r\n                        severity=ValidationSeverity.ERROR,\r\n                        message=f\"Required field '{field}' is missing\",\r\n                        current_value=value,\r\n                        expected=\"Non-empty value\"\r\n                    ))\r\n\r\n        # Check numeric fields\r\n        for field in self.NUMERIC_FIELDS:\r\n            if field in row.index:\r\n                value = row[field]\r\n                if pd.notna(value):\r\n                    try:\r\n                        num_val = float(value)\r\n                        # Check for negative where positive expected\r\n                        if field in self.POSITIVE_FIELDS and num_val \u003C 0:\r\n                            issues.append(ValidationIssue(\r\n                                field=field,\r\n                                record_id=record_id,\r\n                                category=ValidationCategory.INVALID_VALUE,\r\n                                severity=ValidationSeverity.ERROR,\r\n                                message=f\"Negative value in '{field}'\",\r\n                                current_value=value,\r\n                                expected=\"Positive number\"\r\n                            ))\r\n\r\n                        # Check for outliers\r\n                        if self._stats and field in self._stats:\r\n                            stats = self._stats[field]\r\n                            lower = stats['q1'] - self.OUTLIER_THRESHOLD * stats['iqr']\r\n                            upper = stats['q3'] + self.OUTLIER_THRESHOLD * stats['iqr']\r\n\r\n                            if num_val \u003C lower or num_val > upper:\r\n                                issues.append(ValidationIssue(\r\n                                    field=field,\r\n                                    record_id=record_id,\r\n                                    category=ValidationCategory.OUTLIER,\r\n                                    severity=ValidationSeverity.WARNING,\r\n                                    message=f\"Outlier value in '{field}'\",\r\n                                    current_value=value,\r\n                                    expected=f\"Between {lower:.2f} and {upper:.2f}\"\r\n                                ))\r\n\r\n                    except (ValueError, TypeError):\r\n                        issues.append(ValidationIssue(\r\n                            field=field,\r\n                            record_id=record_id,\r\n                            category=ValidationCategory.INVALID_VALUE,\r\n                            severity=ValidationSeverity.ERROR,\r\n                            message=f\"Non-numeric value in '{field}'\",\r\n                            current_value=value,\r\n                            expected=\"Numeric value\"\r\n                        ))\r\n\r\n        # Check work item code format\r\n        if 'work_item_code' in row.index:\r\n            code = row['work_item_code']\r\n            if pd.notna(code) and not self._valid_code_format(str(code)):\r\n                issues.append(ValidationIssue(\r\n                    field='work_item_code',\r\n                    record_id=record_id,\r\n                    category=ValidationCategory.FORMAT,\r\n                    severity=ValidationSeverity.INFO,\r\n                    message=\"Non-standard code format\",\r\n                    current_value=code,\r\n                    expected=\"CATEGORY-NUMBER format\"\r\n                ))\r\n\r\n        return issues\r\n\r\n    def _valid_code_format(self, code: str) -> bool:\r\n        \"\"\"Check if code follows expected format.\"\"\"\r\n        # Expect format like \"CONC-001\" or \"EXCV-DEEP-002\"\r\n        parts = code.split('-')\r\n        return len(parts) >= 2 and parts[0].isalpha()\r\n\r\n    def validate_estimate(self,\r\n                          items: List[Dict[str, Any]],\r\n                          check_against_cwicr: bool = True) -> ValidationResult:\r\n        \"\"\"Validate estimate items.\"\"\"\r\n\r\n        issues = []\r\n        valid_count = 0\r\n\r\n        for i, item in enumerate(items):\r\n            record_id = str(i)\r\n            item_issues = []\r\n\r\n            # Check required fields\r\n            code = item.get('work_item_code', item.get('code'))\r\n            if not code:\r\n                item_issues.append(ValidationIssue(\r\n                    field='work_item_code',\r\n                    record_id=record_id,\r\n                    category=ValidationCategory.MISSING_DATA,\r\n                    severity=ValidationSeverity.ERROR,\r\n                    message=\"Missing work item code\",\r\n                    current_value=None,\r\n                    expected=\"Valid work item code\"\r\n                ))\r\n\r\n            # Check quantity\r\n            qty = item.get('quantity', 0)\r\n            if qty \u003C= 0:\r\n                item_issues.append(ValidationIssue(\r\n                    field='quantity',\r\n                    record_id=record_id,\r\n                    category=ValidationCategory.INVALID_VALUE,\r\n                    severity=ValidationSeverity.ERROR,\r\n                    message=\"Invalid quantity\",\r\n                    current_value=qty,\r\n                    expected=\"Positive number\"\r\n                ))\r\n\r\n            # Check against CWICR reference\r\n            if check_against_cwicr and self.reference is not None and code:\r\n                if 'work_item_code' in self.reference.columns:\r\n                    if code not in self.reference['work_item_code'].values:\r\n                        item_issues.append(ValidationIssue(\r\n                            field='work_item_code',\r\n                            record_id=record_id,\r\n                            category=ValidationCategory.INVALID_VALUE,\r\n                            severity=ValidationSeverity.WARNING,\r\n                            message=f\"Work item code not found in CWICR: {code}\",\r\n                            current_value=code,\r\n                            expected=\"Valid CWICR code\"\r\n                        ))\r\n\r\n            issues.extend(item_issues)\r\n\r\n            if not any(i.severity == ValidationSeverity.ERROR for i in item_issues):\r\n                valid_count += 1\r\n\r\n        return ValidationResult(\r\n            total_records=len(items),\r\n            valid_records=valid_count,\r\n            issues=issues,\r\n            error_count=sum(1 for i in issues if i.severity == ValidationSeverity.ERROR),\r\n            warning_count=sum(1 for i in issues if i.severity == ValidationSeverity.WARNING),\r\n            info_count=sum(1 for i in issues if i.severity == ValidationSeverity.INFO),\r\n            validation_date=datetime.now(),\r\n            passed=all(i.severity != ValidationSeverity.ERROR for i in issues)\r\n        )\r\n\r\n    def get_data_quality_score(self, result: ValidationResult) -> Dict[str, Any]:\r\n        \"\"\"Calculate data quality score.\"\"\"\r\n\r\n        if result.total_records == 0:\r\n            return {'score': 0, 'grade': 'N\u002FA'}\r\n\r\n        # Weighted scoring\r\n        error_weight = 10\r\n        warning_weight = 3\r\n        info_weight = 1\r\n\r\n        total_deductions = (\r\n            result.error_count * error_weight +\r\n            result.warning_count * warning_weight +\r\n            result.info_count * info_weight\r\n        )\r\n\r\n        max_deductions = result.total_records * error_weight\r\n        score = max(0, 100 - (total_deductions \u002F max_deductions * 100)) if max_deductions > 0 else 100\r\n\r\n        # Assign grade\r\n        if score >= 95:\r\n            grade = 'A'\r\n        elif score >= 85:\r\n            grade = 'B'\r\n        elif score >= 75:\r\n            grade = 'C'\r\n        elif score >= 60:\r\n            grade = 'D'\r\n        else:\r\n            grade = 'F'\r\n\r\n        return {\r\n            'score': round(score, 1),\r\n            'grade': grade,\r\n            'total_records': result.total_records,\r\n            'valid_records': result.valid_records,\r\n            'error_count': result.error_count,\r\n            'warning_count': result.warning_count\r\n        }\r\n\r\n    def export_validation_report(self,\r\n                                  result: ValidationResult,\r\n                                  output_path: str) -> str:\r\n        \"\"\"Export validation report to Excel.\"\"\"\r\n\r\n        with pd.ExcelWriter(output_path, engine='openpyxl') as writer:\r\n            # Summary\r\n            quality = self.get_data_quality_score(result)\r\n            summary_df = pd.DataFrame([{\r\n                'Total Records': result.total_records,\r\n                'Valid Records': result.valid_records,\r\n                'Errors': result.error_count,\r\n                'Warnings': result.warning_count,\r\n                'Info': result.info_count,\r\n                'Quality Score': quality['score'],\r\n                'Grade': quality['grade'],\r\n                'Validation Date': result.validation_date,\r\n                'Passed': result.passed\r\n            }])\r\n            summary_df.to_excel(writer, sheet_name='Summary', index=False)\r\n\r\n            # Issues\r\n            if result.issues:\r\n                issues_df = pd.DataFrame([\r\n                    {\r\n                        'Record': i.record_id,\r\n                        'Field': i.field,\r\n                        'Category': i.category.value,\r\n                        'Severity': i.severity.value,\r\n                        'Message': i.message,\r\n                        'Current Value': str(i.current_value),\r\n                        'Expected': i.expected\r\n                    }\r\n                    for i in result.issues\r\n                ])\r\n                issues_df.to_excel(writer, sheet_name='Issues', index=False)\r\n\r\n        return output_path\r\n```\r\n\r\n## Quick Start\r\n\r\n```python\r\n# Load CWICR reference\r\ncwicr = pd.read_parquet(\"ddc_cwicr_en.parquet\")\r\n\r\n# Initialize validator\r\nvalidator = CWICRDataValidator(cwicr)\r\n\r\n# Validate estimate\r\nitems = [\r\n    {'work_item_code': 'CONC-001', 'quantity': 100},\r\n    {'work_item_code': 'INVALID-CODE', 'quantity': -5}\r\n]\r\n\r\nresult = validator.validate_estimate(items)\r\nprint(f\"Passed: {result.passed}\")\r\nprint(f\"Errors: {result.error_count}\")\r\n```\r\n\r\n## Common Use Cases\r\n\r\n### 1. Data Quality Score\r\n```python\r\nquality = validator.get_data_quality_score(result)\r\nprint(f\"Score: {quality['score']} ({quality['grade']})\")\r\n```\r\n\r\n### 2. Validate DataFrame\r\n```python\r\nimport_df = pd.read_excel(\"estimate_import.xlsx\")\r\nresult = validator.validate_dataframe(import_df)\r\n\r\nfor issue in result.issues:\r\n    if issue.severity == ValidationSeverity.ERROR:\r\n        print(f\"ERROR: {issue.message}\")\r\n```\r\n\r\n### 3. Export Report\r\n```python\r\nvalidator.export_validation_report(result, \"validation_report.xlsx\")\r\n```\r\n\r\n## Resources\r\n- **GitHub**: [OpenConstructionEstimate-DDC-CWICR](https:\u002F\u002Fgithub.com\u002Fdatadrivenconstruction\u002FOpenConstructionEstimate-DDC-CWICR)\r\n- **DDC Book**: Chapter 2.1 - Data Quality Management\r\n",{"slug":5,"description":7,"display_name":37},"CWICR Data Validator","main","e7e336ac628c3e3bd771aff1aaa469c93778f94a","skills\u002Fdatadrivenconstruction\u002Fcwicr-data-validator",0,"https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del","2026-08-23 22:39:37",{"source":8,"skillName":5,"commands":46,"prompt":56,"zipAvailable":57},[47,50,53],{"tool":48,"command":49},"npx","npx skills add kbarbel640-del\u002Fskills --skill cwicr-data-validator",{"tool":51,"command":52},"bunx","bunx skills add kbarbel640-del\u002Fskills --skill cwicr-data-validator",{"tool":54,"command":55},"pnpm","pnpm dlx skills add kbarbel640-del\u002Fskills --skill cwicr-data-validator","安装技能，技能地址：https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fdatadrivenconstruction\u002Fcwicr-data-validator\u002FSKILL.md",true,[59],{"path":60,"resourceType":61,"resourceTypeDesc":62,"sizeBytes":63,"sourceHtmlUrl":64},"_meta.json","REFERENCE","参考文档",308,"https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fdatadrivenconstruction\u002Fcwicr-data-validator\u002F_meta.json",[66,95,124,145,151,171],{"skillNo":67,"name":68,"nameZh":6,"description":69,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":70,"categories":71,"keywords":75,"publishedAt":32},"01M6G8J55967D6281ACMKAKTEQ","crowd-prompting","A marketplace where AI agents improve prompts, system instructions, tool descriptions, and other text-based content with domain expertise from real-world operations — and earn tokens for valuable contributions.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzzadrian\u002Fcrowd-prompting\u002FSKILL.md",[72],{"categoryNo":73,"categoryName":74,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000036","内容创作者",[76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94],"crowd prompting","prompting","api","with","tokens","domain","world","prompts","operations","instructions","text","tool","system","improve","productivity","marketplace","https","content","ai agents",{"skillNo":96,"name":97,"nameZh":6,"description":98,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":99,"categories":100,"keywords":106,"publishedAt":32},"01M6G8J0PKPAYVET6BWE8W4C26","dilemm-ai","Compete in DilemmAI, the prisoner's dilemma AI arena at dilemm.ai. Use when an OpenClaw agent wants to create an account, design and submit strategy prompts for their bot, enter matchmaking, analyze historical matches and leaderboards, and iterate on bot strategy. Requires headless browser automation (no API available yet).","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzyori\u002Fdilemm-ai\u002FSKILL.md",[101],{"categoryNo":102,"categoryName":103,"taxonomyType":104,"taxonomyTypeDesc":105},"01JBSEEDCAT000000000000023","Skill与智能体开发","SCENARIO","使用场景",[107,78,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,83,123],"dilemm ai","arena","analyze","browser","openclaw agent","create","automation","browser automation","bot","openclaw","design","strategy","headless","compete","submit","matchmaking","use",{"skillNo":125,"name":126,"nameZh":6,"description":127,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":128,"categories":129,"keywords":133,"publishedAt":32},"01M6G8HWAF3K9SW7NP3Z83CPJ6","arxiv-paper-reviews","Interact with arXiv Crawler API to fetch papers, read reviews, and submit comments. Use when working with arXiv papers, fetching paper lists by date\u002Fcategory\u002Finterest, viewing paper details with comments, or submitting paper reviews via API at http:\u002F\u002F150.158.152.82:8000.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzxrys\u002Fweak-accept\u002FSKILL.md",[130],{"categoryNo":131,"categoryName":132,"taxonomyType":104,"taxonomyTypeDesc":105},"01JBSEEDCAT000000000000021","资料检索与研究",[134,135,136,137,138,139,140,79,141,121,123,142,78,143,144],"arxiv paper reviews","reviews","arxiv","paper","crawler","papers","comments","fetch","read","http","via",{"skillNo":146,"name":126,"nameZh":6,"description":127,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":147,"categories":148,"keywords":150,"publishedAt":32},"01M6G8HQJ7N7WMY6YV7D7SSJ16","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzxrys\u002Farxiv-paper-reviews\u002FSKILL.md",[149],{"categoryNo":131,"categoryName":132,"taxonomyType":104,"taxonomyTypeDesc":105},[137,136,135,134,144,139,140,79,138,121,123,141,142,78,143],{"skillNo":152,"name":153,"nameZh":6,"description":154,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":155,"categories":156,"keywords":157,"publishedAt":32},"01M6G8HM95Y64JNVFC2HN4E2BG","git-summary","Get a quick summary of the current Git repository including status, recent commits, branches, and contributors.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fzweack\u002Fgit-summary\u002FSKILL.md",[],[158,159,160,161,162,163,164,165,166,167,168,169,170],"git","summary","git summary","commits","repository","win32","status","linux","contributors","true","quick","get","branches",{"skillNo":172,"name":173,"nameZh":6,"description":174,"descriptionZh":6,"repositoryFullName":8,"ownerLogin":9,"ownerAvatarUrl":10,"ownerAccountNo":11,"ownerDisplayName":6,"repositoryNo":12,"starsCount":13,"licenseSpdxId":14,"licenseFileUrl":6,"sourceHtmlUrl":175,"categories":176,"keywords":181,"publishedAt":32},"01M6G8HGEGF05CNEJCGVYFSC4R","agent-constitution","Interact with AgentConstitution governance contracts on Base Sepolia. Check compliance, read rules, log actions, query governance state.","https:\u002F\u002Fgithub.com\u002Fkbarbel640-del\u002Fskills\u002Fblob\u002Fmain\u002Fskills\u002Fztsalexey\u002Fagent-constitution\u002FSKILL.md",[177,178],{"categoryNo":102,"categoryName":103,"taxonomyType":104,"taxonomyTypeDesc":105},{"categoryNo":179,"categoryName":180,"taxonomyType":20,"taxonomyTypeDesc":21},"01JBSEEDCAT000000000000037","律师与法务",[182,183,184,185,79,186,187,38,188,142,29,189,190,191,192,92,193,194,195],"agent constitution","constitution","contracts","tree","query","base","log","Rules","projects","compliance","actions","state","governance","github"]