up
This commit is contained in:
commit
500147aac7
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
*.csv
|
||||
*.csv.gz
|
||||
node_modules/
|
||||
19
README.md
Normal file
19
README.md
Normal file
@ -0,0 +1,19 @@
|
||||
# DVF
|
||||
https://www.data.gouv.fr/fr/datasets/demandes-de-valeurs-foncieres/
|
||||
curl -L https://www.data.gouv.fr/fr/datasets/r/78348f03-a11c-4a6b-b8db-2acf4fee81b1 -o dvf/2023.csv
|
||||
curl -L https://www.data.gouv.fr/fr/datasets/r/87038926-fb31-4959-b2ae-7a24321c599a -o dvf/2022.csv
|
||||
curl -L https://www.data.gouv.fr/fr/datasets/r/817204ac-2202-4b4a-98e7-4184d154d98c -o dvf/2021.csv
|
||||
curl -L https://www.data.gouv.fr/fr/datasets/r/90a98de0-f562-4328-aa16-fe0dd1dca60f -o dvf/2020.csv
|
||||
curl -L https://www.data.gouv.fr/fr/datasets/r/3004168d-bec4-44d9-a781-ef16f41856a2 -o dvf/2019.csv
|
||||
|
||||
# GEODVF
|
||||
https://files.data.gouv.fr/geo-dvf/latest/csv/
|
||||
curl https://files.data.gouv.fr/geo-dvf/latest/csv/2023/full.csv.gz -o geodvf/2023.csv.gz
|
||||
curl https://files.data.gouv.fr/geo-dvf/latest/csv/2022/full.csv.gz -o geodvf/2022.csv.gz
|
||||
curl https://files.data.gouv.fr/geo-dvf/latest/csv/2021/full.csv.gz -o geodvf/2021.csv.gz
|
||||
curl https://files.data.gouv.fr/geo-dvf/latest/csv/2020/full.csv.gz -o geodvf/2020.csv.gz
|
||||
curl https://files.data.gouv.fr/geo-dvf/latest/csv/2019/full.csv.gz -o geodvf/2019.csv.gz
|
||||
|
||||
# run
|
||||
node parse.js dvf/2023.csv
|
||||
...
|
||||
45
dvf/dvf.json
Normal file
45
dvf/dvf.json
Normal file
@ -0,0 +1,45 @@
|
||||
{
|
||||
'Identifiant de document': '',
|
||||
'Reference document': '',
|
||||
'1 Articles CGI': '',
|
||||
'2 Articles CGI': '',
|
||||
'3 Articles CGI': '',
|
||||
'4 Articles CGI': '',
|
||||
'5 Articles CGI': '',
|
||||
'No disposition': '000001',
|
||||
'Date mutation': '05/01/2023',
|
||||
'Nature mutation': 'Vente',
|
||||
'Valeur fonciere': '1070000,00',
|
||||
'No voie': '184',
|
||||
'B/T/Q': '',
|
||||
'Type de voie': 'ALL',
|
||||
'Code voie': '0124',
|
||||
Voie: 'DES HETRES',
|
||||
'Code postal': '1630',
|
||||
Commune: 'ST-GENIS-POUILLY',
|
||||
'Code departement': '01',
|
||||
'Code commune': '354',
|
||||
'Prefixe de section': '',
|
||||
Section: 'BD',
|
||||
'No plan': '334',
|
||||
'No Volume': '',
|
||||
'1er lot': '29',
|
||||
'Surface Carrez du 1er lot': '',
|
||||
'2eme lot': '',
|
||||
'Surface Carrez du 2eme lot': '',
|
||||
'3eme lot': '',
|
||||
'Surface Carrez du 3eme lot': '',
|
||||
'4eme lot': '',
|
||||
'Surface Carrez du 4eme lot': '',
|
||||
'5eme lot': '',
|
||||
'Surface Carrez du 5eme lot': '',
|
||||
'Nombre de lots': '1',
|
||||
'Code type local': '3',
|
||||
'Type local': 'Dépendance',
|
||||
'Identifiant local': '',
|
||||
'Surface reelle bati': '0',
|
||||
'Nombre pieces principales': '0',
|
||||
'Nature culture': '',
|
||||
'Nature culture speciale': '',
|
||||
'Surface terrain': ''
|
||||
}
|
||||
42
geodvf/geodvf.json
Normal file
42
geodvf/geodvf.json
Normal file
@ -0,0 +1,42 @@
|
||||
{
|
||||
id_mutation: '2023-1',
|
||||
date_mutation: '2023-01-05',
|
||||
numero_disposition: '000001',
|
||||
nature_mutation: 'Vente',
|
||||
valeur_fonciere: '1070000',
|
||||
adresse_numero: '184',
|
||||
adresse_suffixe: '',
|
||||
adresse_nom_voie: 'ALL DES HETRES',
|
||||
adresse_code_voie: '0124',
|
||||
code_postal: '01630',
|
||||
code_commune: '01354',
|
||||
nom_commune: 'Saint-Genis-Pouilly',
|
||||
code_departement: '01',
|
||||
ancien_code_commune: '',
|
||||
ancien_nom_commune: '',
|
||||
id_parcelle: '01354000BD0334',
|
||||
ancien_id_parcelle: '',
|
||||
numero_volume: '',
|
||||
lot1_numero: '29',
|
||||
lot1_surface_carrez: '',
|
||||
lot2_numero: '',
|
||||
lot2_surface_carrez: '',
|
||||
lot3_numero: '',
|
||||
lot3_surface_carrez: '',
|
||||
lot4_numero: '',
|
||||
lot4_surface_carrez: '',
|
||||
lot5_numero: '',
|
||||
lot5_surface_carrez: '',
|
||||
nombre_lots: '1',
|
||||
code_type_local: '3',
|
||||
type_local: 'Dépendance',
|
||||
surface_reelle_bati: '',
|
||||
nombre_pieces_principales: '0',
|
||||
code_nature_culture: '',
|
||||
nature_culture: '',
|
||||
code_nature_culture_speciale: '',
|
||||
nature_culture_speciale: '',
|
||||
surface_terrain: '',
|
||||
longitude: '6.019949',
|
||||
latitude: '46.247458'
|
||||
}
|
||||
147
package-lock.json
generated
Normal file
147
package-lock.json
generated
Normal file
@ -0,0 +1,147 @@
|
||||
{
|
||||
"name": "dvf",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "dvf",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"csv-parse": "^5.5.6",
|
||||
"moment": "^2.30.1",
|
||||
"mysql2": "^3.10.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=21.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "5.5.6",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz",
|
||||
"integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/denque": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
|
||||
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/generate-function": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz",
|
||||
"integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-property": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
"integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"safer-buffer": ">= 2.1.2 < 3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-property": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
|
||||
"integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/long": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
|
||||
"integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "8.0.5",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz",
|
||||
"integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=16.14"
|
||||
}
|
||||
},
|
||||
"node_modules/moment": {
|
||||
"version": "2.30.1",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
|
||||
"integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/mysql2": {
|
||||
"version": "3.10.0",
|
||||
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.10.0.tgz",
|
||||
"integrity": "sha512-qx0mfWYt1DpTPkw8mAcHW/OwqqyNqBLBHvY5IjN8+icIYTjt6znrgYJ+gxqNNRpVknb5Wc/gcCM4XjbCR0j5tw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"denque": "^2.1.0",
|
||||
"generate-function": "^2.3.1",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"long": "^5.2.1",
|
||||
"lru-cache": "^8.0.0",
|
||||
"named-placeholders": "^1.1.3",
|
||||
"seq-queue": "^0.0.5",
|
||||
"sqlstring": "^2.3.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/named-placeholders": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz",
|
||||
"integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"lru-cache": "^7.14.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/named-placeholders/node_modules/lru-cache": {
|
||||
"version": "7.18.3",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
|
||||
"integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
|
||||
"license": "ISC",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/seq-queue": {
|
||||
"version": "0.0.5",
|
||||
"resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
|
||||
"integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q=="
|
||||
},
|
||||
"node_modules/sqlstring": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz",
|
||||
"integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
30
package.json
Normal file
30
package.json
Normal file
@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "dvf",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "ssh://git@gitea.raphaelpiccolo.com:10022/root/dvf.git"
|
||||
},
|
||||
"license": "ISC",
|
||||
"author": "Raphael Piccolo",
|
||||
"type": "module",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"build": "gulp",
|
||||
"cov": "c8 npm run test",
|
||||
"jest": "SILENT=1 node --no-warnings --experimental-vm-modules ./node_modules/.bin/jest",
|
||||
"prepare": "husky",
|
||||
"start": "node server.js",
|
||||
"test": "SILENT=1 mocha --timeout 60000 {lib,test}/**/*.test.js",
|
||||
"watch": "gulp watch"
|
||||
},
|
||||
"dependencies": {
|
||||
"moment": "^2.30.1",
|
||||
"mysql2": "^3.10.0",
|
||||
"csv-parse": "^5.5.6"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=21.0.0"
|
||||
}
|
||||
}
|
||||
38
parse.js
Normal file
38
parse.js
Normal file
@ -0,0 +1,38 @@
|
||||
import { parse } from 'csv-parse';
|
||||
import fs from 'node:fs';
|
||||
import zlib from 'zlib';
|
||||
|
||||
if (process.argv.length != 3) {
|
||||
throw new Error('You should give a project dir');
|
||||
}
|
||||
const file = process.argv[2];
|
||||
const delimiter = (file.match(/\.gz$/)) ? ',' : '|';
|
||||
|
||||
const parser = parse({
|
||||
delimiter,
|
||||
columns: true,
|
||||
});
|
||||
|
||||
let lines = 0;
|
||||
parser.on('readable', function(){
|
||||
let record;
|
||||
while ((record = parser.read()) !== null) {
|
||||
console.log(record);
|
||||
lines++;
|
||||
}
|
||||
});
|
||||
parser.on('error', function(err){
|
||||
console.error(err.message);
|
||||
});
|
||||
parser.on('end', function(){
|
||||
console.log('end', lines);
|
||||
});
|
||||
|
||||
if (file.match(/\.gz$/)) fs.createReadStream(file).pipe(zlib.createGunzip()).pipe(parser);
|
||||
else fs.createReadStream(file).pipe(parser);
|
||||
|
||||
// affiche la progression
|
||||
const interval = setInterval(() => {
|
||||
console.log(`found ${lines} lines`);
|
||||
}, 1000);
|
||||
interval.unref();
|
||||
Loading…
Reference in New Issue
Block a user