You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
hy-screen-2.0/scripts/export-screen2-datasets.mjs

503 lines
22 KiB

#!/usr/bin/env node
import * as esbuild from "esbuild"
import fs from "node:fs"
import path from "node:path"
import vm from "node:vm"
import { fileURLToPath } from "node:url"
import { createRequire } from "node:module"
const require = createRequire(import.meta.url)
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const rootDir = path.resolve(__dirname, "..")
const outputArgIndex = process.argv.findIndex((arg) => arg === "--out")
const outputPath = outputArgIndex >= 0 ? process.argv[outputArgIndex + 1] : ""
const rawPlugin = {
name: "screen2-raw-loader",
setup(build) {
build.onResolve({ filter: /\?raw$/ }, (args) => {
const sourcePath = args.path.replace(/\?raw$/, "")
const absolutePath = sourcePath.startsWith("/public/")
? path.join(rootDir, sourcePath.slice(1))
: path.resolve(args.resolveDir, sourcePath)
return { path: absolutePath, namespace: "screen2-raw" }
})
build.onLoad({ filter: /.*/, namespace: "screen2-raw" }, async (args) => {
const contents = await fs.promises.readFile(args.path, "utf8")
return {
contents: `export default ${JSON.stringify(contents)};`,
loader: "js",
}
})
},
}
const assetPlugin = {
name: "screen2-asset-loader",
setup(build) {
build.onLoad({ filter: /\.(png|jpe?g|webp|gif|svg)$/i }, async (args) => ({
contents: `export default ${JSON.stringify(path.relative(rootDir, args.path))};`,
loader: "js",
}))
},
}
const aliasPlugin = {
name: "screen2-alias",
setup(build) {
build.onResolve({ filter: /^@\// }, (args) => ({
path: resolveImportPath(path.join(rootDir, "src", args.path.slice(2))),
}))
build.onResolve({ filter: /^~@\// }, (args) => ({
path: resolveImportPath(path.join(rootDir, "src", args.path.slice(3))),
}))
build.onResolve({ filter: /^\/public\// }, (args) => ({
path: resolveImportPath(path.join(rootDir, args.path.slice(1))),
}))
},
}
function resolveImportPath(candidate) {
if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) return candidate
for (const ext of [".js", ".json", ".vue"]) {
const withExt = `${candidate}${ext}`
if (fs.existsSync(withExt) && fs.statSync(withExt).isFile()) return withExt
}
const indexPath = path.join(candidate, "index.js")
if (fs.existsSync(indexPath) && fs.statSync(indexPath).isFile()) return indexPath
return candidate
}
const entry = `
import * as gd from "./src/views/gdMap/livestockData.js";
import * as home from "./src/views/home/data.js";
import * as pasture from "./src/views/pastureManagement/data.js";
import * as yak from "./src/views/yakManagement/data.js";
import * as balance from "./src/views/grassLivestockBalance/data.js";
import * as industry from "./src/views/yakIndustryChain/data.js";
import * as ecology from "./src/views/ecologicalProtection/data.js";
export default { gd, home, pasture, yak, balance, industry, ecology };
`
const buildResult = await esbuild.build({
stdin: {
contents: entry,
resolveDir: rootDir,
sourcefile: "screen2-dataset-entry.js",
loader: "js",
},
bundle: true,
write: false,
platform: "node",
format: "cjs",
target: "node18",
plugins: [rawPlugin, assetPlugin, aliasPlugin],
logLevel: "silent",
})
const moduleStub = { exports: {} }
vm.runInNewContext(buildResult.outputFiles[0].text, {
module: moduleStub,
exports: moduleStub.exports,
require,
console,
process,
Buffer,
setTimeout,
clearTimeout,
})
const exported = moduleStub.exports.default || moduleStub.exports
const datasetDefinitions = []
const configDefinitions = []
const industrySubjects = []
const VALUE_KEYS = [
"value",
"count",
"number",
"yakCount",
"stock",
"ratio",
"areaWanMu",
"areaRange",
"yakNumber",
"bearingCapacity",
"livestockCapacity",
"overloadRate",
"capacity",
"projectCount",
"subjectFeatureCount",
"lengthKm",
"peopleCount",
"householdCount",
"ecologyJobs",
"percent",
]
function addRows(datasetKey, pageCode, title, rows, options = {}) {
if (!Array.isArray(rows)) return
datasetDefinitions.push({
datasetKey,
pageCode,
moduleCode: options.moduleCode || pageCode,
datasetType: options.datasetType || "rows",
sourceType: options.sourceType || "frontend_seed",
title,
payload: options.payload || {},
rows,
})
}
function addConfig(configKey, groupKey, title, payload) {
if (payload === undefined) return
configDefinitions.push({ configKey, groupKey, title, payload })
}
function objectToRows(source = {}, labels = {}) {
return Object.entries(source || {}).map(([key, value]) => ({
key,
name: labels[key] || key,
value,
}))
}
function uniqueRows(rows = [], keyBuilder) {
const seen = new Set()
return rows.filter((row, index) => {
const key = keyBuilder(row, index)
if (seen.has(key)) return false
seen.add(key)
return true
})
}
function collectDefinitions(data) {
const { gd, home, pasture, yak, balance, industry, ecology } = data
addRows("gdMap.core", "gdMap", "gdMap 核心指标", objectToRows(gd.homeCore, {
herdsmanCount: "牧户数量",
grasslandArea: "草地面积",
wetArea: "湿地面积",
yakCount: "牦牛数量",
projectCount: "工程数量",
baseCount: "基地数量",
}))
addRows("gdMap.grasslandWet", "gdMap", "草地湿地乡镇分布", gd.grasslandWetRows)
addRows("gdMap.yakForecast", "gdMap", "牦牛月度预测", gd.yakForecastRows)
addRows("gdMap.townRecognition", "gdMap", "乡镇遥感识别", gd.townRecognitionRows)
addRows("gdMap.grasslandYak", "gdMap", "草地牦牛乡镇分布", gd.grasslandYakRows)
addRows("gdMap.herdStructure", "gdMap", "畜群结构", gd.herdStructureRows)
addRows("gdMap.tradeImpact", "gdMap", "交易影响", gd.tradeImpactRows)
addRows("gdMap.stockForecast", "gdMap", "存栏预测", gd.stockForecastRows)
addRows("gdMap.supportIndustry", "gdMap", "支撑产业", gd.supportIndustryRows)
addRows("gdMap.warning", "gdMap", "预警统计", gd.warningRows)
addConfig("gdMap.stockTotals", "gdMap", "牦牛存栏总量配置", {
YAK_STOCK_TOTAL: gd.YAK_STOCK_TOTAL,
YAK_DRONE_RECOGNITION_TOTAL: gd.YAK_DRONE_RECOGNITION_TOTAL,
YAK_EXPERT_CORRECTION: gd.YAK_EXPERT_CORRECTION,
recognitionTotal: gd.recognitionTotal,
expertCorrection: gd.expertCorrection,
tradeDeduction: gd.tradeDeduction,
estimatedStock: gd.estimatedStock,
})
addRows("home.topStats", "home", "首页顶部指标", home.homeTopStats)
addRows("home.heroStats", "home", "首页主视觉指标", home.homeHeroStats)
addRows("home.resourceComposition", "home", "首页资源构成", home.resourceCompositionRows)
addRows("home.businessMatrix", "home", "首页业务矩阵", home.businessMatrixRows)
addRows("home.townSynergy", "home", "首页乡镇协同", home.townSynergyRows)
addRows("home.yakTown", "home", "首页牦牛乡镇排行", home.yakTownRows)
addRows("home.trade", "home", "首页交易结构", home.tradeRows)
addRows("home.support", "home", "首页支撑产业", home.supportRows)
addRows("home.dataFlow", "home", "首页数据流", home.dataFlowRows)
addRows("home.centerOrbit", "home", "首页中心节点", home.centerOrbitNodes)
addRows("pasture.core", "pastureManagement", "牧户核心指标", pasture.pastureCoreRows)
addRows("pasture.topStats", "pastureManagement", "牧户顶部指标", pasture.pastureTopStats)
addRows("pasture.town", "pastureManagement", "牧户乡镇分布", pasture.pastureTownRows)
addRows("pasture.online", "pastureManagement", "牧户在线率", pasture.pastureOnlineRows)
addRows("pasture.structure", "pastureManagement", "牧户年龄性别结构", pasture.pastureStructureRows)
addRows("pasture.age", "pastureManagement", "牧户年龄结构", pasture.pastureAgeRows)
addRows("pasture.gender", "pastureManagement", "牧户性别结构", pasture.pastureGenderRows)
addRows("pasture.list", "pastureManagement", "牧户示例列表", pasture.pastureListRows)
addRows("pasture.grassland", "pastureManagement", "牧户草场资源", pasture.pastureGrasslandRows)
addRows("pasture.season", "pastureManagement", "牧户季节草场", pasture.pastureSeasonRows)
addRows("pasture.map", "pastureManagement", "牧户地图统计", pasture.pastureMapRows)
addRows("yak.town", "yakManagement", "牦牛乡镇分布", yak.townRows)
addRows("yak.structureBaseline", "yakManagement", "牦牛结构基准", yak.yakStructureBaselineRows)
addRows("yak.structureRatio", "yakManagement", "牦牛结构比例", yak.yakStructureRatioRows)
addRows("yak.structureRows", "yakManagement", "牦牛结构数量", yak.structureRows)
addRows("yak.tradeImpact", "yakManagement", "牦牛交易影响", yak.tradeImpactRows)
addRows("yak.genderAge", "yakManagement", "牦牛年龄性别", yak.genderAgeRows)
addRows("yak.forecastPeriods", "yakManagement", "牦牛预测周期", yak.forecastPeriods)
addRows("yak.forecastFactors", "yakManagement", "牦牛预测因素", yak.forecastFactorRows)
addRows("yak.forecastRows", "yakManagement", "牦牛月度预测", yak.yakForecastRows)
addConfig("yak.stockTotals", "yakManagement", "牦牛管理总量配置", {
YAK_STOCK_TOTAL: yak.YAK_STOCK_TOTAL,
recognitionTotal: yak.recognitionTotal,
expertCorrection: yak.expertCorrection,
tradeDeduction: yak.tradeDeduction,
estimatedStock: yak.estimatedStock,
})
addRows("balance.county", "grassLivestockBalance", "草畜平衡县级数据", [balance.countyBalanceRow])
addRows("balance.town", "grassLivestockBalance", "草畜平衡乡镇数据", balance.balanceTownRows)
addRows("balance.fallbackRows", "grassLivestockBalance", "草畜平衡兜底数据", balance.balanceFallbackRows)
addConfig("balance.params", "grassLivestockBalance", "草畜平衡计算参数", balance.BALANCE_CALCULATION_PARAMS)
addConfig("balance.countyTarget", "grassLivestockBalance", "草畜平衡县级对象", balance.COUNTY_TARGET)
addConfig("balance.riskLevels", "grassLivestockBalance", "草畜平衡风险等级", balance.RISK_LEVELS)
addRows("industry.topStats", "yakIndustryChain", "产业链顶部指标", industry.industryTopStats)
addRows("industry.overview", "yakIndustryChain", "产业链总览指标", industry.industryOverviewRows)
addRows("industry.stageComposition", "yakIndustryChain", "产业链阶段构成", industry.stageCompositionRows)
addRows("industry.stageRows", "yakIndustryChain", "产业链阶段数据", industry.industryStageRows)
addRows("industry.breedingLevel", "yakIndustryChain", "养殖水平", industry.breedingLevelRows)
addRows("industry.forageSecurity", "yakIndustryChain", "饲草保障", industry.forageSecurityRows)
addRows("industry.value", "yakIndustryChain", "产业价值", industry.industryValueRows)
addRows("industry.processingAbility", "yakIndustryChain", "加工能力", industry.processingAbilityRows)
addRows("industry.serviceCapability", "yakIndustryChain", "服务能力", industry.serviceCapabilityRows)
addRows("industry.financial", "yakIndustryChain", "金融服务", industry.industryFinancialRows)
addRows("industry.supportPoints", "yakIndustryChain", "饲草储备点位", industry.supportPointRows)
addRows("industry.processingPoints", "yakIndustryChain", "加工屠宰点位", industry.processingPointRows)
addRows("industry.points", "yakIndustryChain", "产业链点位", industry.industryPointRows)
addRows("industry.list", "yakIndustryChain", "产业链主体列表", industry.industryChainListRows)
addConfig("industry.stageDefs", "yakIndustryChain", "产业链阶段定义", industry.industryStageDefs)
addConfig("industry.datasetMeta", "yakIndustryChain", "产业链数据集信息", {
generatedAt: industry.industryDatasetGeneratedAt,
})
const subjectRows = uniqueRows(industry.industryChainListRows || [], (row, index) =>
String(row.id || row.key || row.name || index),
)
subjectRows.forEach((row, index) => industrySubjects.push({ row, index }))
addRows("ecology.parkTown", "ecologicalProtection", "国家公园乡镇数据", ecology.parkTownRows)
addRows("ecology.parkCore", "ecologicalProtection", "国家公园核心指标", ecology.parkCoreRows)
addRows("ecology.parkTopStats", "ecologicalProtection", "国家公园顶部指标", ecology.parkTopStats)
addRows("ecology.restorationTopStats", "ecologicalProtection", "生态修复顶部指标", ecology.restorationTopStats)
addRows("ecology.engineeringAnalysisTopStats", "ecologicalProtection", "工程分析顶部指标", ecology.engineeringAnalysisTopStats)
addRows("ecology.engineeringProjectFeatures", "ecologicalProtection", "工程分析项目点", ecology.engineeringAnalysisProjectFeatures)
addConfig("ecology.menu", "ecologicalProtection", "生态保护菜单", ecology.ecologicalProtectionMenu)
addConfig("ecology.parkPanels", "ecologicalProtection", "国家公园面板", ecology.parkPanels)
addConfig("ecology.restorationPanels", "ecologicalProtection", "生态修复面板", ecology.restorationPanels)
addConfig("ecology.engineeringAnalysisPanels", "ecologicalProtection", "工程分析面板", ecology.engineeringAnalysisPanels)
addConfig("ecology.centerMapNotes", "ecologicalProtection", "生态保护地图说明", ecology.centerMapNotes)
}
function pickValueKey(row = {}) {
for (const key of VALUE_KEYS) {
const value = row[key]
if (value !== undefined && value !== null && value !== "" && Number.isFinite(Number(value))) {
return key
}
}
const firstNumeric = Object.keys(row).find((key) => Number.isFinite(Number(row[key])))
return firstNumeric || ""
}
function normalizeRow(datasetKey, row, index) {
const valueKey = pickValueKey(row)
const value = valueKey ? row[valueKey] : undefined
const rowKey = row.key ?? row.id ?? row.code ?? row.areaCode ?? row.area_code ?? row.name ?? row.label ?? `${datasetKey}-${index + 1}`
return {
datasetKey,
rowKey: String(rowKey).slice(0, 255),
valueKey,
areaCode: row.areaCode ?? row.area_code ?? row.townCode ?? row.town_code ?? row.code ?? null,
areaName: row.areaName ?? row.area_name ?? row.townName ?? row.town ?? row.zoneName ?? null,
name: row.name ?? row.zh ?? row.label ?? row.areaName ?? row.area_name ?? row.zoneName ?? row.projectName ?? null,
valueNum: value !== undefined && value !== null && value !== "" && Number.isFinite(Number(value)) ? Number(value) : null,
valueText: value !== undefined && value !== null && value !== "" && !Number.isFinite(Number(value)) ? String(value) : null,
unit: row.unit ?? null,
percentNum: row.percent !== undefined && row.percent !== null && Number.isFinite(Number(row.percent)) ? Number(row.percent) : null,
extraText: row.extra ?? row.note ?? row.description ?? null,
period: row.year ?? row.month ?? row.period ?? row.label ?? null,
lng: row.lng ?? row.longitude ?? null,
lat: row.lat ?? row.latitude ?? null,
sortOrder: index,
attrs: row,
}
}
function normalizeIndustrySubject(row, index) {
const subjectKey = String(row.id || row.key || row.name || `industry-subject-${index + 1}`).slice(0, 255)
const valueKey = pickValueKey(row)
return {
subjectKey,
stageKey: row.stageKey ?? row.stage_key ?? row.industryStage ?? null,
categoryCode: row.categoryCode ?? row.category_code ?? null,
categoryName: row.categoryName ?? row.category_name ?? null,
name: row.name ?? row.subjectName ?? subjectKey,
town: row.town ?? row.townName ?? row.areaName ?? null,
areaName: row.areaName ?? row.area_name ?? null,
address: row.address ?? row.location ?? row.detailAddress ?? null,
lng: row.lng ?? row.longitude ?? null,
lat: row.lat ?? row.latitude ?? null,
valueNum: valueKey && Number.isFinite(Number(row[valueKey])) ? Number(row[valueKey]) : null,
unit: row.unit ?? null,
sortOrder: index,
attrs: row,
sourceMeta: {
sourceType: "frontend_seed",
categoryCode: row.categoryCode ?? null,
generatedAt: exported.industry.industryDatasetGeneratedAt || null,
},
}
}
function sqlText(value) {
if (value === undefined || value === null) return "NULL"
return `'${String(value).replace(/'/g, "''")}'`
}
function sqlNumber(value) {
if (value === undefined || value === null || value === "") return "NULL"
const number = Number(value)
return Number.isFinite(number) ? String(number) : "NULL"
}
function sqlBoolean(value) {
return value ? "true" : "false"
}
function sqlJson(value) {
return `${sqlText(JSON.stringify(value ?? {}))}::jsonb`
}
function buildSql() {
collectDefinitions(exported)
const datasetKeys = datasetDefinitions.map((item) => item.datasetKey)
const configKeys = configDefinitions.map((item) => item.configKey)
const lines = [
"-- Generated by screenV2.0/scripts/export-screen2-datasets.mjs",
"-- Source: current Screen2.0 frontend static data modules.",
"",
"BEGIN;",
"",
]
if (datasetKeys.length) {
lines.push(`DELETE FROM public.screen2_dataset_row WHERE dataset_key IN (${datasetKeys.map(sqlText).join(", ")});`)
}
if (industrySubjects.length) {
lines.push("DELETE FROM public.screen2_industry_subject WHERE source_type = 'frontend_seed';")
}
if (configKeys.length) {
lines.push(`DELETE FROM public.screen2_config WHERE config_key IN (${configKeys.map(sqlText).join(", ")});`)
}
lines.push("")
datasetDefinitions.forEach((dataset, datasetIndex) => {
lines.push(`INSERT INTO public.screen2_dataset (
dataset_key, page_code, module_code, dataset_type, source_type, title, sort_order, enabled, payload, update_time
) VALUES (
${sqlText(dataset.datasetKey)}, ${sqlText(dataset.pageCode)}, ${sqlText(dataset.moduleCode)},
${sqlText(dataset.datasetType)}, ${sqlText(dataset.sourceType)}, ${sqlText(dataset.title)},
${datasetIndex}, true, ${sqlJson(dataset.payload)}, now()
)
ON CONFLICT (dataset_key) DO UPDATE SET
page_code = EXCLUDED.page_code,
module_code = EXCLUDED.module_code,
dataset_type = EXCLUDED.dataset_type,
source_type = EXCLUDED.source_type,
title = EXCLUDED.title,
sort_order = EXCLUDED.sort_order,
enabled = EXCLUDED.enabled,
payload = EXCLUDED.payload,
update_time = now();`)
dataset.rows.map((row, index) => normalizeRow(dataset.datasetKey, row, index)).forEach((row) => {
lines.push(`INSERT INTO public.screen2_dataset_row (
dataset_key, row_key, value_key, area_code, area_name, name, value_num, value_text,
unit, percent_num, extra_text, period, lng, lat, sort_order, enabled, attrs, update_time
) VALUES (
${sqlText(row.datasetKey)}, ${sqlText(row.rowKey)}, ${sqlText(row.valueKey || null)},
${sqlText(row.areaCode)}, ${sqlText(row.areaName)}, ${sqlText(row.name)}, ${sqlNumber(row.valueNum)},
${sqlText(row.valueText)}, ${sqlText(row.unit)}, ${sqlNumber(row.percentNum)}, ${sqlText(row.extraText)},
${sqlText(row.period)}, ${sqlNumber(row.lng)}, ${sqlNumber(row.lat)}, ${row.sortOrder},
true, ${sqlJson(row.attrs)}, now()
)
ON CONFLICT (dataset_key, row_key) DO UPDATE SET
value_key = EXCLUDED.value_key,
area_code = EXCLUDED.area_code,
area_name = EXCLUDED.area_name,
name = EXCLUDED.name,
value_num = EXCLUDED.value_num,
value_text = EXCLUDED.value_text,
unit = EXCLUDED.unit,
percent_num = EXCLUDED.percent_num,
extra_text = EXCLUDED.extra_text,
period = EXCLUDED.period,
lng = EXCLUDED.lng,
lat = EXCLUDED.lat,
sort_order = EXCLUDED.sort_order,
enabled = EXCLUDED.enabled,
attrs = EXCLUDED.attrs,
update_time = now();`)
})
})
configDefinitions.forEach((config) => {
lines.push(`INSERT INTO public.screen2_config (
config_key, group_key, title, payload, enabled, update_time
) VALUES (
${sqlText(config.configKey)}, ${sqlText(config.groupKey)}, ${sqlText(config.title)},
${sqlJson(config.payload)}, true, now()
)
ON CONFLICT (config_key) DO UPDATE SET
group_key = EXCLUDED.group_key,
title = EXCLUDED.title,
payload = EXCLUDED.payload,
enabled = EXCLUDED.enabled,
update_time = now();`)
})
industrySubjects.map(({ row, index }) => normalizeIndustrySubject(row, index)).forEach((subject) => {
lines.push(`INSERT INTO public.screen2_industry_subject (
subject_key, stage_key, category_code, category_name, name, town, area_name, address,
lng, lat, value_num, unit, sort_order, source_type, enabled, attrs, source_meta, update_time
) VALUES (
${sqlText(subject.subjectKey)}, ${sqlText(subject.stageKey)}, ${sqlText(subject.categoryCode)},
${sqlText(subject.categoryName)}, ${sqlText(subject.name)}, ${sqlText(subject.town)},
${sqlText(subject.areaName)}, ${sqlText(subject.address)}, ${sqlNumber(subject.lng)}, ${sqlNumber(subject.lat)},
${sqlNumber(subject.valueNum)}, ${sqlText(subject.unit)}, ${subject.sortOrder}, 'frontend_seed',
${sqlBoolean(true)}, ${sqlJson(subject.attrs)}, ${sqlJson(subject.sourceMeta)}, now()
)
ON CONFLICT (subject_key) DO UPDATE SET
stage_key = EXCLUDED.stage_key,
category_code = EXCLUDED.category_code,
category_name = EXCLUDED.category_name,
name = EXCLUDED.name,
town = EXCLUDED.town,
area_name = EXCLUDED.area_name,
address = EXCLUDED.address,
lng = EXCLUDED.lng,
lat = EXCLUDED.lat,
value_num = EXCLUDED.value_num,
unit = EXCLUDED.unit,
sort_order = EXCLUDED.sort_order,
source_type = EXCLUDED.source_type,
enabled = EXCLUDED.enabled,
attrs = EXCLUDED.attrs,
source_meta = EXCLUDED.source_meta,
update_time = now();`)
})
lines.push("")
lines.push("COMMIT;")
lines.push("")
return lines.join("\n")
}
const sql = buildSql()
if (outputPath) {
await fs.promises.mkdir(path.dirname(path.resolve(outputPath)), { recursive: true })
await fs.promises.writeFile(path.resolve(outputPath), sql, "utf8")
} else {
process.stdout.write(sql)
}