You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
hy-screen-2.0/scripts/prepare-yak-industry-supple...

574 lines
23 KiB

#!/usr/bin/env python3
"""Prepare supplemental Yak industry-chain data from latest source workbooks."""
from __future__ import annotations
import hashlib
import json
import math
import random
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
import openpyxl
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_MILK_SOURCE = Path("/Users/mocker/Documents/hy/数据清单 2/2026.7.3/2026年奶源站站名.xlsx")
DEFAULT_ARCHIVE = Path("/Users/mocker/Documents/hy/数据清单 2/详细数据清单-归档.xlsx")
DEFAULT_FORAGE_TRADE = Path("/Users/mocker/Documents/hy/数据清单 2/牧草交易中心数据-种草数据.xlsx")
DEFAULT_VILLAGES = ROOT / "public/datas/geojsons/hongyuan-villages.geojson"
DEFAULT_OUTPUT = ROOT / "src/views/yakIndustryChain/yakIndustrySupplementalData.json"
DEFAULT_POINT_GEOJSON = ROOT / "public/datas/yak-industry-chain/yak_industry_points.geojson"
TOWN_NAMES = [
"邛溪镇",
"刷经寺镇",
"安曲镇",
"龙日镇",
"江茸乡",
"查尔玛乡",
"瓦切镇",
"阿木乡",
"麦洼乡",
"色地镇",
]
VILLAGE_ALIASES = {
("邛溪镇", "达格隆村"): ("邛溪镇", "达格龙村"),
("瓦切镇", "色永村"): ("瓦切镇", "色尔永村"),
("色地镇", "让里村"): ("色地镇", "壤里村"),
("龙日镇", "龙日坝村"): ("龙日镇", "四川省龙日种畜场"),
}
SUPPLEMENTAL_CATEGORY_CODES = {
"YC-MILK-SOURCE-STATION",
"YC-FORAGE-TRADE-CENTER",
"YC-PROCESSING-PLANT",
"YC-SLAUGHTERHOUSE",
}
def clean(value: object) -> str:
return "" if value is None else str(value).replace("\n", " ").strip()
def compact(value: object) -> str:
return re.sub(r"\s+", "", clean(value))
def number(value: object) -> float:
if value is None:
return 0.0
if isinstance(value, (int, float)) and math.isfinite(float(value)):
return float(value)
text = clean(value).replace(",", "")
if not text or text in {"", "待完善"}:
return 0.0
matched = re.search(r"-?\d+(?:\.\d+)?", text)
return float(matched.group(0)) if matched else 0.0
def display_number(value: float) -> int | float:
rounded = round(float(value or 0), 2)
return int(rounded) if rounded.is_integer() else rounded
def parse_coordinate(value: object) -> float | None:
if value is None:
return None
if isinstance(value, (int, float)) and math.isfinite(float(value)):
return float(value)
text = clean(value)
if not text:
return None
decimal = re.search(r"([0-9]{2,3})\.(\d+)", text)
if decimal:
return float(decimal.group(0))
degree = re.search(r"([0-9]{2,3})\s*°\s*([0-9.]+)?", text)
if not degree:
plain = re.search(r"([0-9]{2,3})(?:\D+)?([0-9]{2,6})?", text)
if not plain:
return None
deg = float(plain.group(1))
rest = plain.group(2) or ""
else:
deg = float(degree.group(1))
rest = re.sub(r"\D", "", degree.group(2) or "")
if not rest:
return deg
if len(rest) >= 6:
return float(f"{int(deg)}.{rest}")
if len(rest) == 4:
minutes = int(rest[:2])
seconds = int(rest[2:])
if minutes < 60 and seconds < 60:
return deg + minutes / 60 + seconds / 3600
minutes = float(rest)
if minutes < 60:
return deg + minutes / 60
return float(f"{int(deg)}.{rest}")
def parse_lng_lat(row: dict) -> tuple[float | None, float | None]:
lng = parse_coordinate(row.get("经度"))
lat = parse_coordinate(row.get("纬度"))
if is_hongyuan_coordinate(lng, lat):
return round(lng, 8), round(lat, 8)
combined = clean(row.get("经纬度"))
matched = re.search(r"([0-9]{2,3}\.\d+)\s*[,,]\s*([0-9]{2}\.\d+)", combined)
if matched:
lng, lat = float(matched.group(1)), float(matched.group(2))
if is_hongyuan_coordinate(lng, lat):
return round(lng, 8), round(lat, 8)
return lng, lat
def is_hongyuan_coordinate(lng: float | None, lat: float | None) -> bool:
return (
lng is not None
and lat is not None
and 101.5 <= float(lng) <= 103.4
and 31.8 <= float(lat) <= 33.5
)
def rows_from_sheet(path: Path, sheet_name: str) -> list[dict]:
workbook = openpyxl.load_workbook(path, data_only=True)
sheet = workbook[sheet_name]
rows = list(sheet.iter_rows(values_only=True))
header = [clean(value) for value in rows[0]]
output = []
for raw in rows[1:]:
item = {header[index]: raw[index] for index in range(min(len(header), len(raw))) if header[index]}
if any(clean(value) for value in item.values()):
output.append(item)
return output
def parse_archive_section(path: Path, sheet_name: str, category_code: str, category_name: str) -> list[dict]:
workbook = openpyxl.load_workbook(path, data_only=True)
sheet = workbook[sheet_name]
headers: list[str] = []
items = []
index = 1
skip_names = {"企业全称", "主体全称", "农产品检测站", "执法大队"}
for raw in sheet.iter_rows(values_only=True):
row_values = [clean(value) for value in raw]
first = row_values[0] if row_values else ""
if first in {"企业全称", "主体全称"}:
headers = row_values
continue
if not headers or not first or first in skip_names:
continue
row = {headers[col]: raw[col] for col in range(min(len(headers), len(raw))) if headers[col]}
name = clean(row.get("企业全称") or row.get("主体全称"))
if not name or name in skip_names:
continue
if sheet_name == "屠宰厂" and not any(number(row.get(key)) for key in ("设计日屠宰能力 (头)", "年屠宰总量 (头)", "冷冻库容量 (吨)", "年总产值 (万元)")):
continue
lng, lat = parse_lng_lat(row)
metrics = build_archive_metrics(row, sheet_name)
attrs = build_archive_attrs(row, sheet_name)
item = {
"id": f"{category_code}-SUP-{index:03d}",
"name": name,
"categoryCode": category_code,
"categoryName": category_name,
"stageKey": "slaughter-processing",
"stageName": "屠宰与加工",
"address": clean(row.get("详细地址")),
"areaName": normalize_town_from_text(clean(row.get("所属乡镇") or row.get("详细地址") or name)),
"longitude": lng if is_hongyuan_coordinate(lng, lat) else None,
"latitude": lat if is_hongyuan_coordinate(lng, lat) else None,
"status": clean(row.get("运营状态")) or "正常",
"metrics": metrics,
"attributes": attrs,
"sourceSheet": f"详细数据清单-归档.xlsx / {sheet_name}",
}
items.append(item)
index += 1
return items
def build_archive_metrics(row: dict, sheet_name: str) -> dict:
if sheet_name == "加工厂":
keys = [
"设计日处理鲜奶能力 (吨)",
"设计日加工能力 (吨)",
"年加工鲜奶总量 (吨)",
"年加工原料肉总量 (吨)",
"年产值 (万元)",
"带动本地农户数",
"原料奶冷藏能力 (吨)",
"冷库容量 (吨)",
]
else:
keys = ["设计日屠宰能力 (头)", "年屠宰总量 (头)", "冷冻库容量 (吨)", "年总产值 (万元)"]
return {key: row.get(key) for key in keys if clean(row.get(key))}
def build_archive_attrs(row: dict, sheet_name: str) -> dict:
if sheet_name == "加工厂":
keys = ["SC 许可证号", "主要产品", "奶源来源", "原料来源", "产品销售范围"]
else:
keys = ["定点屠宰证号", "环保达标情况"]
return {key: clean(row.get(key)) for key in keys if clean(row.get(key))}
def normalize_town_from_text(text: str) -> str:
compact_text = compact(text)
for town in TOWN_NAMES:
if town in compact_text:
return town
stem = town[:-1]
if len(stem) >= 2 and stem in compact_text:
return town
return ""
def load_village_index(path: Path) -> dict[tuple[str, str], dict]:
geojson = json.loads(path.read_text(encoding="utf-8"))
index = {}
for feature in geojson.get("features", []):
props = feature.get("properties") or {}
town = clean(props.get("town"))
name = clean(props.get("name"))
if town and name:
index[(town, name)] = feature
return index
def normalize_village_key(town: str, village: str) -> tuple[str, str]:
town = clean(town)
village = clean(village)
if village and not village.endswith("") and "种畜场" not in village:
village = f"{village}"
return VILLAGE_ALIASES.get((town, village), (town, village))
def parse_area_candidates(raw_area: str, village_index: dict[tuple[str, str], dict]) -> list[tuple[str, str, dict]]:
text = compact(raw_area)
candidates = []
for (town, village), feature in village_index.items():
village_aliases = {village, village.replace("色尔永", "色永"), village.replace("达格龙", "达格隆"), village.replace("壤里", "让里")}
if any(alias and alias in text for alias in village_aliases):
if town in text or normalize_town_from_text(text) in {"", town} or any(alias in text for alias in village_aliases):
candidates.append((town, village, feature))
if candidates:
return dedupe_candidates(candidates)
town = normalize_town_from_text(text)
parts = re.split(r"[、,,/;;]+", text)
for part in parts:
village_match = re.search(r"([\u4e00-\u9fa5A-Za-z0-9]+?村)", part)
if not village_match:
continue
key = normalize_village_key(town, village_match.group(1))
feature = village_index.get(key)
if feature:
candidates.append((*key, feature))
return dedupe_candidates(candidates)
def dedupe_candidates(candidates: list[tuple[str, str, dict]]) -> list[tuple[str, str, dict]]:
seen = set()
output = []
for town, village, feature in candidates:
key = (town, village)
if key in seen:
continue
seen.add(key)
output.append((town, village, feature))
return output
def parse_milk_source_stations(path: Path, village_index: dict[tuple[str, str], dict]) -> list[dict]:
workbook = openpyxl.load_workbook(path, data_only=True)
sheet = workbook.active
items = []
for row in sheet.iter_rows(values_only=True):
name = clean(row[0] if len(row) > 0 else "")
area = clean(row[1] if len(row) > 1 else "")
if not name or name == "站名" or "奶源中心" in name or not area:
continue
candidates = parse_area_candidates(area, village_index)
if candidates:
seed_index = stable_seed(name) % len(candidates)
town, village, feature = candidates[seed_index]
lng, lat = random_point_in_feature(feature, f"{name}-{town}-{village}")
match_text = f"村界随机点:{town} · {village}"
else:
town = normalize_town_from_text(area)
village = ""
lng = lat = None
match_text = "未匹配到红原县村界"
items.append({
"id": f"YC-MILK-SOURCE-STATION-SUP-{len(items) + 1:03d}",
"name": f"{name}奶源站",
"shortName": name,
"categoryCode": "YC-MILK-SOURCE-STATION",
"categoryName": "奶源站",
"stageKey": "milk-source-station",
"stageName": "奶源站",
"areaName": town,
"town": town,
"village": village,
"address": area,
"longitude": lng,
"latitude": lat,
"status": "2026年奶源站",
"metrics": {"站点数量": 1},
"attributes": {"所属乡镇、村": area, "点位生成": match_text},
"sourceSheet": "2026年奶源站站名.xlsx / Sheet1",
})
return items
def stable_seed(text: str) -> int:
return int(hashlib.md5(text.encode("utf-8")).hexdigest()[:8], 16)
def random_point_in_feature(feature: dict, seed_text: str) -> tuple[float, float]:
polygons = feature_polygons(feature)
rng = random.Random(stable_seed(seed_text))
rings = [ring for polygon in polygons for ring in polygon[:1] if ring]
xs = [point[0] for ring in rings for point in ring]
ys = [point[1] for ring in rings for point in ring]
bbox = (min(xs), min(ys), max(xs), max(ys))
for _ in range(2000):
x = rng.uniform(bbox[0], bbox[2])
y = rng.uniform(bbox[1], bbox[3])
if point_in_feature((x, y), polygons):
return round(x, 8), round(y, 8)
centroid = approximate_centroid(rings[0])
return round(centroid[0], 8), round(centroid[1], 8)
def feature_polygons(feature: dict) -> list[list[list[tuple[float, float]]]]:
geometry = feature.get("geometry") or {}
if geometry.get("type") == "Polygon":
return [[[(float(x), float(y)) for x, y, *_ in ring] for ring in geometry.get("coordinates", [])]]
if geometry.get("type") == "MultiPolygon":
return [
[[(float(x), float(y)) for x, y, *_ in ring] for ring in polygon]
for polygon in geometry.get("coordinates", [])
]
return []
def point_in_feature(point: tuple[float, float], polygons: list[list[list[tuple[float, float]]]]) -> bool:
for polygon in polygons:
if not polygon:
continue
if not point_in_ring(point, polygon[0]):
continue
if any(point_in_ring(point, hole) for hole in polygon[1:]):
continue
return True
return False
def point_in_ring(point: tuple[float, float], ring: list[tuple[float, float]]) -> bool:
x, y = point
inside = False
count = len(ring)
if count < 3:
return False
j = count - 1
for i in range(count):
xi, yi = ring[i]
xj, yj = ring[j]
intersects = (yi > y) != (yj > y) and x < (xj - xi) * (y - yi) / ((yj - yi) or 1e-12) + xi
if intersects:
inside = not inside
j = i
return inside
def approximate_centroid(ring: list[tuple[float, float]]) -> tuple[float, float]:
if not ring:
return 0, 0
return sum(point[0] for point in ring) / len(ring), sum(point[1] for point in ring) / len(ring)
def parse_forage_trade(path: Path) -> tuple[list[dict], dict]:
workbook = openpyxl.load_workbook(path, data_only=True)
sheet = workbook.active
rows = []
current_year = ""
year_total_row = None
combined_total_row = None
for raw in sheet.iter_rows(min_row=4, values_only=True):
values = list(raw)
if not any(clean(value) for value in values):
continue
first_cell = clean(values[0] if len(values) > 0 else "")
if first_cell in {"年度", "红原县2026年草种销售、采购情况", "红原县2025—2026年总的草种销售、采购情况"}:
current_year = ""
continue
if first_cell in {"总计", "合计"}:
if first_cell == "合计":
combined_total_row = values
else:
year_total_row = values
continue
if first_cell:
year_match = re.search(r"20\d{2}", first_cell)
current_year = year_match.group(0) if year_match else first_cell
seller = clean(values[1] if len(values) > 1 else "")
buyer = clean(values[8] if len(values) > 8 else "")
if seller == "销售主体" or buyer == "采购主体" or clean(values[3] if len(values) > 3 else "") == "饲草":
continue
if not seller and not buyer:
continue
item = {
"key": f"forage-trade-row-{len(rows) + 1}",
"year": current_year or "2025",
"seller": seller,
"plantAreaMu": number(values[2] if len(values) > 2 else None),
"salesForageTon": number(values[3] if len(values) > 3 else None),
"salesSeedTon": number(values[4] if len(values) > 4 else None),
"foragePriceYuanPerTon": number(values[5] if len(values) > 5 else None),
"seedPriceYuanPerKg": number(values[6] if len(values) > 6 else None),
"salesAmountWanYuan": number(values[7] if len(values) > 7 else None),
"buyer": buyer,
"purchaseForageTon": number(values[9] if len(values) > 9 else None),
"seedVarietyText": clean(values[10] if len(values) > 10 else ""),
"purchaseForagePriceYuanPerTon": number(values[11] if len(values) > 11 else None),
"purchaseSeedPriceYuanPerKg": clean(values[12] if len(values) > 12 else ""),
"purchaseAmountWanYuan": number(values[13] if len(values) > 13 else None),
}
rows.append(item)
total_row = combined_total_row or year_total_row
metrics = {
"sellerCount": len({row["seller"] for row in rows if row["seller"]}),
"buyerCount": len({row["buyer"] for row in rows if row["buyer"]}),
"plantAreaMu": number(total_row[2]) if total_row else sum(row["plantAreaMu"] for row in rows),
"salesForageTon": number(total_row[3]) if total_row else sum(row["salesForageTon"] for row in rows),
"salesSeedTon": number(total_row[4]) if total_row else sum(row["salesSeedTon"] for row in rows),
"salesAmountWanYuan": number(total_row[7]) if total_row else sum(row["salesAmountWanYuan"] for row in rows),
"purchaseForageTon": number(total_row[9]) if total_row else sum(row["purchaseForageTon"] for row in rows),
"purchaseSeedKg": number(total_row[10]) if total_row else 0,
"purchaseAmountWanYuan": number(total_row[13]) if total_row else sum(row["purchaseAmountWanYuan"] for row in rows),
}
items = [{
"id": "YC-FORAGE-TRADE-CENTER-SUP-001",
"name": "瓦切牧草交易中心",
"categoryCode": "YC-FORAGE-TRADE-CENTER",
"categoryName": "牧草交易中心",
"stageKey": "forage-trade",
"stageName": "牧草交易",
"areaName": "瓦切镇",
"town": "瓦切镇",
"address": "瓦切镇达俄村",
"longitude": 102.6014834,
"latitude": 33.09604707,
"status": "2025年交易数据",
"metrics": {
"种草面积 (亩)": display_number(metrics["plantAreaMu"]),
"饲草销售量 (吨)": display_number(metrics["salesForageTon"]),
"草种销售量 (吨)": display_number(metrics["salesSeedTon"]),
"销售金额 (万元)": display_number(metrics["salesAmountWanYuan"]),
"采购金额 (万元)": display_number(metrics["purchaseAmountWanYuan"]),
},
"attributes": {
"销售主体数": str(metrics["sellerCount"]),
"采购主体数": str(metrics["buyerCount"]),
"数据年度": "2025-2026",
},
"sourceSheet": "牧草交易中心数据-种草数据.xlsx / Sheet1",
}]
return items, {"rows": rows, "metrics": {key: display_number(value) for key, value in metrics.items()}}
def make_category(category_code: str, category_name: str, stage_key: str, items: list[dict], replace: bool = False) -> dict:
category = {
"categoryCode": category_code,
"categoryName": category_name,
"industryStage": stage_key,
"items": items,
}
if replace:
category["replaceCategory"] = True
return category
def update_point_geojson(point_path: Path, categories: list[dict]) -> None:
geojson = json.loads(point_path.read_text(encoding="utf-8"))
existing = [
feature
for feature in geojson.get("features", [])
if feature.get("properties", {}).get("categoryCode") not in SUPPLEMENTAL_CATEGORY_CODES
]
for category in categories:
for item in category.get("items", []):
lng = item.get("longitude")
lat = item.get("latitude")
if not is_hongyuan_coordinate(lng, lat):
continue
props = {
**item,
"categoryCode": category["categoryCode"],
"categoryName": category["categoryName"],
"stageKey": category["industryStage"],
"stageName": item.get("stageName") or category["categoryName"],
"supplemental": True,
}
existing.append({
"type": "Feature",
"properties": props,
"geometry": {
"type": "Point",
"coordinates": [round(float(lng), 8), round(float(lat), 8)],
},
})
geojson["features"] = existing
point_path.write_text(json.dumps(geojson, ensure_ascii=False, indent=2), encoding="utf-8")
def main() -> None:
milk_source = Path(sys.argv[1]).expanduser() if len(sys.argv) > 1 else DEFAULT_MILK_SOURCE
archive = Path(sys.argv[2]).expanduser() if len(sys.argv) > 2 else DEFAULT_ARCHIVE
forage_trade_source = Path(sys.argv[3]).expanduser() if len(sys.argv) > 3 else DEFAULT_FORAGE_TRADE
village_index = load_village_index(DEFAULT_VILLAGES)
milk_items = parse_milk_source_stations(milk_source, village_index)
processing_items = parse_archive_section(archive, "加工厂", "YC-PROCESSING-PLANT", "加工厂")
slaughter_items = parse_archive_section(archive, "屠宰厂", "YC-SLAUGHTERHOUSE", "屠宰厂")
forage_trade_items, forage_trade = parse_forage_trade(forage_trade_source)
categories = [
make_category("YC-MILK-SOURCE-STATION", "奶源站", "milk-source-station", milk_items),
make_category("YC-PROCESSING-PLANT", "加工厂", "slaughter-processing", processing_items, replace=True),
make_category("YC-SLAUGHTERHOUSE", "屠宰厂", "slaughter-processing", slaughter_items, replace=True),
make_category("YC-FORAGE-TRADE-CENTER", "牧草交易中心", "forage-trade", forage_trade_items),
]
payload = {
"generatedAt": datetime.now(timezone.utc).isoformat(),
"sources": {
"milkSourceStations": str(milk_source),
"archive": str(archive),
"forageTrade": str(forage_trade_source),
"villages": str(DEFAULT_VILLAGES),
},
"categories": categories,
"forageTrade": forage_trade,
"summary": {
"milkSourceStationCount": len(milk_items),
"milkSourceLocatedCount": sum(1 for item in milk_items if is_hongyuan_coordinate(item.get("longitude"), item.get("latitude"))),
"processingPlantCount": len(processing_items),
"slaughterhouseCount": len(slaughter_items),
"forageTradeCenterCount": len(forage_trade_items),
},
}
DEFAULT_OUTPUT.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
update_point_geojson(DEFAULT_POINT_GEOJSON, categories)
print(json.dumps(payload["summary"], ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()