up

2024-06-01 23:21:29 +02:00
commit 500147aac7
7 changed files with 324 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,3 @@
+*.csv
+*.csv.gz
+node_modules/
--- a/README.md
+++ b/README.md
@@ -0,0 +1,19 @@
+# DVF
+https://www.data.gouv.fr/fr/datasets/demandes-de-valeurs-foncieres/
+curl -L https://www.data.gouv.fr/fr/datasets/r/78348f03-a11c-4a6b-b8db-2acf4fee81b1 -o dvf/2023.csv
+curl -L https://www.data.gouv.fr/fr/datasets/r/87038926-fb31-4959-b2ae-7a24321c599a -o dvf/2022.csv
+curl -L https://www.data.gouv.fr/fr/datasets/r/817204ac-2202-4b4a-98e7-4184d154d98c -o dvf/2021.csv
+curl -L https://www.data.gouv.fr/fr/datasets/r/90a98de0-f562-4328-aa16-fe0dd1dca60f -o dvf/2020.csv
+curl -L https://www.data.gouv.fr/fr/datasets/r/3004168d-bec4-44d9-a781-ef16f41856a2 -o dvf/2019.csv
+
+# GEODVF
+https://files.data.gouv.fr/geo-dvf/latest/csv/
+curl https://files.data.gouv.fr/geo-dvf/latest/csv/2023/full.csv.gz -o geodvf/2023.csv.gz
+curl https://files.data.gouv.fr/geo-dvf/latest/csv/2022/full.csv.gz -o geodvf/2022.csv.gz
+curl https://files.data.gouv.fr/geo-dvf/latest/csv/2021/full.csv.gz -o geodvf/2021.csv.gz
+curl https://files.data.gouv.fr/geo-dvf/latest/csv/2020/full.csv.gz -o geodvf/2020.csv.gz
+curl https://files.data.gouv.fr/geo-dvf/latest/csv/2019/full.csv.gz -o geodvf/2019.csv.gz
+
+# run
+node parse.js dvf/2023.csv
+...
--- a/dvf/dvf.json
+++ b/dvf/dvf.json
@@ -0,0 +1,45 @@
+{
+  'Identifiant de document': '',
+  'Reference document': '',
+  '1 Articles CGI': '',
+  '2 Articles CGI': '',
+  '3 Articles CGI': '',
+  '4 Articles CGI': '',
+  '5 Articles CGI': '',
+  'No disposition': '000001',
+  'Date mutation': '05/01/2023',
+  'Nature mutation': 'Vente',
+  'Valeur fonciere': '1070000,00',
+  'No voie': '184',
+  'B/T/Q': '',
+  'Type de voie': 'ALL',
+  'Code voie': '0124',
+  Voie: 'DES HETRES',
+  'Code postal': '1630',
+  Commune: 'ST-GENIS-POUILLY',
+  'Code departement': '01',
+  'Code commune': '354',
+  'Prefixe de section': '',
+  Section: 'BD',
+  'No plan': '334',
+  'No Volume': '',
+  '1er lot': '29',
+  'Surface Carrez du 1er lot': '',
+  '2eme lot': '',
+  'Surface Carrez du 2eme lot': '',
+  '3eme lot': '',
+  'Surface Carrez du 3eme lot': '',
+  '4eme lot': '',
+  'Surface Carrez du 4eme lot': '',
+  '5eme lot': '',
+  'Surface Carrez du 5eme lot': '',
+  'Nombre de lots': '1',
+  'Code type local': '3',
+  'Type local': 'Dépendance',
+  'Identifiant local': '',
+  'Surface reelle bati': '0',
+  'Nombre pieces principales': '0',
+  'Nature culture': '',
+  'Nature culture speciale': '',
+  'Surface terrain': ''
+}
--- a/geodvf/geodvf.json
+++ b/geodvf/geodvf.json
@@ -0,0 +1,42 @@
+{
+  id_mutation: '2023-1',
+  date_mutation: '2023-01-05',
+  numero_disposition: '000001',
+  nature_mutation: 'Vente',
+  valeur_fonciere: '1070000',
+  adresse_numero: '184',
+  adresse_suffixe: '',
+  adresse_nom_voie: 'ALL DES HETRES',
+  adresse_code_voie: '0124',
+  code_postal: '01630',
+  code_commune: '01354',
+  nom_commune: 'Saint-Genis-Pouilly',
+  code_departement: '01',
+  ancien_code_commune: '',
+  ancien_nom_commune: '',
+  id_parcelle: '01354000BD0334',
+  ancien_id_parcelle: '',
+  numero_volume: '',
+  lot1_numero: '29',
+  lot1_surface_carrez: '',
+  lot2_numero: '',
+  lot2_surface_carrez: '',
+  lot3_numero: '',
+  lot3_surface_carrez: '',
+  lot4_numero: '',
+  lot4_surface_carrez: '',
+  lot5_numero: '',
+  lot5_surface_carrez: '',
+  nombre_lots: '1',
+  code_type_local: '3',
+  type_local: 'Dépendance',
+  surface_reelle_bati: '',
+  nombre_pieces_principales: '0',
+  code_nature_culture: '',
+  nature_culture: '',
+  code_nature_culture_speciale: '',
+  nature_culture_speciale: '',
+  surface_terrain: '',
+  longitude: '6.019949',
+  latitude: '46.247458'
+}
--- a/package-lock.json
+++ b/package-lock.json
@@ -0,0 +1,147 @@
+{
+    "name": "dvf",
+    "version": "1.0.0",
+    "lockfileVersion": 3,
+    "requires": true,
+    "packages": {
+        "": {
+            "name": "dvf",
+            "version": "1.0.0",
+            "license": "ISC",
+            "dependencies": {
+                "csv-parse": "^5.5.6",
+                "moment": "^2.30.1",
+                "mysql2": "^3.10.0"
+            },
+            "engines": {
+                "node": ">=21.0.0"
+            }
+        },
+        "node_modules/csv-parse": {
+            "version": "5.5.6",
+            "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.6.tgz",
+            "integrity": "sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A==",
+            "license": "MIT"
+        },
+        "node_modules/denque": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+            "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+            "license": "Apache-2.0",
+            "engines": {
+                "node": ">=0.10"
+            }
+        },
+        "node_modules/generate-function": {
+            "version": "2.3.1",
+            "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz",
+            "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==",
+            "license": "MIT",
+            "dependencies": {
+                "is-property": "^1.0.2"
+            }
+        },
+        "node_modules/iconv-lite": {
+            "version": "0.6.3",
+            "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+            "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+            "license": "MIT",
+            "dependencies": {
+                "safer-buffer": ">= 2.1.2 < 3.0.0"
+            },
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
+        "node_modules/is-property": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
+            "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==",
+            "license": "MIT"
+        },
+        "node_modules/long": {
+            "version": "5.2.3",
+            "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
+            "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==",
+            "license": "Apache-2.0"
+        },
+        "node_modules/lru-cache": {
+            "version": "8.0.5",
+            "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz",
+            "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==",
+            "license": "ISC",
+            "engines": {
+                "node": ">=16.14"
+            }
+        },
+        "node_modules/moment": {
+            "version": "2.30.1",
+            "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
+            "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
+            "license": "MIT",
+            "engines": {
+                "node": "*"
+            }
+        },
+        "node_modules/mysql2": {
+            "version": "3.10.0",
+            "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.10.0.tgz",
+            "integrity": "sha512-qx0mfWYt1DpTPkw8mAcHW/OwqqyNqBLBHvY5IjN8+icIYTjt6znrgYJ+gxqNNRpVknb5Wc/gcCM4XjbCR0j5tw==",
+            "license": "MIT",
+            "dependencies": {
+                "denque": "^2.1.0",
+                "generate-function": "^2.3.1",
+                "iconv-lite": "^0.6.3",
+                "long": "^5.2.1",
+                "lru-cache": "^8.0.0",
+                "named-placeholders": "^1.1.3",
+                "seq-queue": "^0.0.5",
+                "sqlstring": "^2.3.2"
+            },
+            "engines": {
+                "node": ">= 8.0"
+            }
+        },
+        "node_modules/named-placeholders": {
+            "version": "1.1.3",
+            "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.3.tgz",
+            "integrity": "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==",
+            "license": "MIT",
+            "dependencies": {
+                "lru-cache": "^7.14.1"
+            },
+            "engines": {
+                "node": ">=12.0.0"
+            }
+        },
+        "node_modules/named-placeholders/node_modules/lru-cache": {
+            "version": "7.18.3",
+            "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz",
+            "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==",
+            "license": "ISC",
+            "engines": {
+                "node": ">=12"
+            }
+        },
+        "node_modules/safer-buffer": {
+            "version": "2.1.2",
+            "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+            "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+            "license": "MIT"
+        },
+        "node_modules/seq-queue": {
+            "version": "0.0.5",
+            "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
+            "integrity": "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q=="
+        },
+        "node_modules/sqlstring": {
+            "version": "2.3.3",
+            "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.3.tgz",
+            "integrity": "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==",
+            "license": "MIT",
+            "engines": {
+                "node": ">= 0.6"
+            }
+        }
+    }
+}
--- a/package.json
+++ b/package.json
@@ -0,0 +1,30 @@
+{
+    "name": "dvf",
+    "version": "1.0.0",
+    "description": "",
+    "repository": {
+        "type": "git",
+        "url": "ssh://git@gitea.raphaelpiccolo.com:10022/root/dvf.git"
+    },
+    "license": "ISC",
+    "author": "Raphael Piccolo",
+    "type": "module",
+    "main": "server.js",
+    "scripts": {
+        "build": "gulp",
+        "cov": "c8 npm run test",
+        "jest": "SILENT=1 node --no-warnings --experimental-vm-modules ./node_modules/.bin/jest",
+        "prepare": "husky",
+        "start": "node server.js",
+        "test": "SILENT=1 mocha --timeout 60000 {lib,test}/**/*.test.js",
+        "watch": "gulp watch"
+    },
+    "dependencies": {
+        "moment": "^2.30.1",
+        "mysql2": "^3.10.0",
+        "csv-parse": "^5.5.6"
+    },
+    "engines": {
+        "node": ">=21.0.0"
+    }
+}
--- a/parse.js
+++ b/parse.js
@@ -0,0 +1,38 @@
+import { parse } from 'csv-parse';
+import fs from 'node:fs';
+import zlib from 'zlib';
+
+if (process.argv.length != 3) {
+    throw new Error('You should give a project dir');
+}
+const file = process.argv[2];
+const delimiter = (file.match(/\.gz$/)) ? ',' : '|';
+
+const parser = parse({
+    delimiter,
+    columns: true,
+});
+
+let lines = 0;
+parser.on('readable', function(){
+    let record;
+    while ((record = parser.read()) !== null) {
+        console.log(record);
+        lines++;
+    }
+});
+parser.on('error', function(err){
+    console.error(err.message);
+});
+parser.on('end', function(){
+    console.log('end', lines);
+});
+
+if (file.match(/\.gz$/)) fs.createReadStream(file).pipe(zlib.createGunzip()).pipe(parser);
+else fs.createReadStream(file).pipe(parser);
+
+// affiche la progression
+const interval = setInterval(() => {
+    console.log(`found ${lines} lines`);
+}, 1000);
+interval.unref();