root
10af5a50e5
parse.js/sync.js/lib/dotenv.js -> .ts (run via tsx), ajout tsconfig.json, eslint config TS-enabled (parser tseslint + resolver typescript), devDeps typescript/typescript-eslint/tsx/eslint-import-resolver-typescript, scripts package.json en tsx + typecheck, .lintstagedrc *.ts, README en tsx. Fix securite: escapeId sur les identifiants de colonnes dans sync.ts (au lieu de backticks bruts) pour ne pas casser/injecter via un en-tete CSV distant. Suppression du todo.txt audit (finding escapeId traite). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
67 lines
1.8 KiB
TypeScript
67 lines
1.8 KiB
TypeScript
import { parse } from 'csv-parse';
|
|
import fs from 'node:fs';
|
|
import zlib from 'node:zlib';
|
|
import mysql from 'mysql2';
|
|
import './lib/dotenv.ts';
|
|
|
|
if (process.argv.length != 3) {
|
|
throw new Error('You should give a project dir');
|
|
}
|
|
const file = process.argv[2];
|
|
|
|
// csv parser
|
|
const delimiter = (file.match(/\.gz$/)) ? ',' : '|';
|
|
const parser = parse({
|
|
delimiter,
|
|
columns: true,
|
|
});
|
|
|
|
// mysql to escape string
|
|
const connectionString = process.env.MYSQL;
|
|
if (!connectionString) {
|
|
throw new Error('MYSQL environment variable not set');
|
|
}
|
|
|
|
const connection = mysql.createConnection(process.env.MYSQL);
|
|
|
|
// generate sql
|
|
const columns: string[] = [];
|
|
const flushBatch = (batch: unknown[][]) => {
|
|
if (batch.length === 0) return;
|
|
|
|
const values = batch.map(row => `(${row.map(val => connection.escape(val)).join(', ')})`).join(', ');
|
|
const sql = `INSERT INTO dvf (${columns.join(', ')}) VALUES ${values};`;
|
|
|
|
console.log(sql);
|
|
};
|
|
|
|
// quand quelques ligne de csv sont parsées on les assemble puis on genere le sql
|
|
parser.on('readable', () =>{
|
|
const batch = [];
|
|
|
|
let record = parser.read();
|
|
while (record !== null) {
|
|
// console.log(record);
|
|
|
|
// get columns and values to insert and escape them for sql
|
|
if (columns.length === 0) {
|
|
columns.push(...Object.keys(record).map(col => connection.escapeId(col)));
|
|
}
|
|
const values = Object.values(record);
|
|
batch.push(values);
|
|
record = parser.read();
|
|
}
|
|
|
|
flushBatch(batch);
|
|
});
|
|
parser.on('error', (err) =>{
|
|
console.error(err.message);
|
|
});
|
|
parser.on('end', () =>{
|
|
connection.end();
|
|
});
|
|
|
|
// ouvre le fichier, et le décompresse si besoin
|
|
if (file.match(/\.gz$/)) fs.createReadStream(file).pipe(zlib.createGunzip()).pipe(parser);
|
|
else fs.createReadStream(file).pipe(parser);
|