Store network inside a cached file

Create script for updating the network, suggest suitable refs for stops
that lack one.

back/data/endpoints.js

@@ -0,0 +1,115 @@
+const unzip = require('unzip-stream');
+const csv = require('csv-parse');
+const request = require('request');
+const requestp = require('request-promise-native');
+
+const overpassEndpoint = 'https://lz4.overpass-api.de/api/interpreter';
+
+/**
+ * Submit an Overpass query.
+ *
+ * @async
+ * @param query Query in Overpass QL.
+ * @return Results as provided by the endpoint.
+ */
+const queryOverpass = query => requestp.post(
+    overpassEndpoint,
+    {form: 'data=' + query}
+);
+
+exports.queryOverpass = queryOverpass;
+
+/**
+ * Process a CSV stream to extract passings.
+ *
+ * @private
+ * @param csvStream Stream containing CSV data.
+ * @param callback See fetchTamRealtime for a description of the callback.
+ */
+const processTamPassingStream = (csvStream, callback) =>
+{
+    const parser = csv({
+        delimiter: ';',
+    });
+
+    const rowStream = csvStream.pipe(parser);
+
+    rowStream.on('readable', () =>
+    {
+        let row;
+
+        while (row = rowStream.read())
+        {
+            if (row.length === 0 || row[0] === 'course')
+            {
+                // Ignore les lignes invalides et l’en-tête
+                continue;
+            }
+
+            callback(null, {
+                course: row[0],
+                stopCode: row[1],
+                stopId: row[2],
+                stopName: row[3],
+                routeShortName: row[4],
+                tripHeadsign: row[5],
+                directionId: row[6],
+                departureTime: row[7],
+                isTheorical: row[8],
+                delaySec: row[9],
+                destArCode: row[10],
+            });
+        }
+    });
+
+    rowStream.on('end', () => callback(null, null));
+    rowStream.on('error', err => callback(err));
+};
+
+const tamRealtimeEndpoint = 'http://data.montpellier3m.fr/node/10732/download';
+
+/**
+ * Fetch realtime passings for the current day across the network.
+ *
+ * @param callback Called for each passing during parsing. First argument will
+ * be non-null only if an error occurred. Second argument will contain passings
+ * or be null if the end was reached.
+ */
+const fetchTamRealtime = (callback) =>
+{
+    const csvStream = request(tamRealtimeEndpoint);
+    processTamPassingStream(csvStream, callback);
+};
+
+exports.fetchTamRealtime = fetchTamRealtime;
+
+const tamTheoreticalEndpoint =
+    'http://data.montpellier3m.fr/node/10731/download';
+const tamTheoreticalFileName = 'offre_du_jour.csv';
+
+/**
+ * Fetch theoretical passings for the current day across the network.
+ *
+ * @param callback Called for each passing during parsing. First argument will
+ * be non-null only if an error occurred. Second argument will contain passings
+ * or be null if the end was reached.
+ */
+const fetchTamTheoretical = (callback) =>
+{
+    const fileStream = request(tamTheoreticalEndpoint).pipe(unzip.Parse());
+
+    fileStream.on('entry', entry =>
+    {
+        if (entry.type !== 'File' || entry.path !== tamTheoreticalFileName)
+        {
+            entry.autodrain();
+            return;
+        }
+
+        processTamPassingStream(entry, callback);
+    });
+
+    fileStream.on('error', err => callback(err));
+};
+
+exports.fetchTamTheoretical = fetchTamTheoretical;

back/data/network.js

@@ -1,15 +1,132 @@
-const requestp = require('request-promise-native');
 const geolib = require('geolib');
 
-const {makeCached} = require('../util');
+const {choosePlural, joinSentence} = require('../util');
+const {queryOverpass, fetchTamTheoretical} = require('./endpoints');
 
-const OVERPASS_ENDPOINT = 'https://lz4.overpass-api.de/api/interpreter';
+const osmViewNode = 'https://www.openstreetmap.org/node';
 
-const fetch = makeCached(async (lineRefs) =>
+/**
+ * Create a link to remotely add tags into JOSM.
+ *
+ * @param objectId Identifier for the object to add the tags to.
+ * @param tags Tags to add.
+ * @return Link for remotely adding the tags.
+ */
+const josmAddTagToNode = (objectId, tags) =>
+    'http://127.0.0.1:8111/load_object?' + [
+        `objects=n${objectId}`,
+        'new_layer=false',
+        'addtags=' + tags.join('%7C')
+    ].join('&');
+
+/**
+ * Use theoretical passings data to guess which lines use which stops in which
+ * direction.
+ *
+ * This is used for suggesting possible stop IDs for stops that don’t have
+ * one in OSM.
+ *
+ * @return Map containing for each stop its abbreviated name, the lines that
+ * use it and in which directions it is used.
+ */
+const fetchStopsRefAssociations = () => new Promise((res, rej) =>
+{
+    const stops = {};
+
+    fetchTamTheoretical((err, row) =>
+    {
+        if (err)
+        {
+            rej(err);
+            return;
+        }
+
+        if (!row)
+        {
+            res(stops);
+            return;
+        }
+
+        if (!(row.stopId in stops))
+        {
+            stops[row.stopId] = {
+                name: row.stopName,
+                lines: new Set([row.routeShortName]),
+                directions: new Set([row.tripHeadsign]),
+            };
+        }
+        else
+        {
+            const stop = stops[row.stopId];
+
+            if (stop.name !== row.stopName)
+            {
+                console.warn(`Stop ${row.stopId} has multiple names: \
+“${row.stopName}” and “${stop.name}”. Only the first one will be considered.`);
+            }
+
+            stop.lines.add(row.routeShortName);
+            stop.directions.add(row.tripHeadsign);
+        }
+    });
+});
+
+// Mapping for abbreviations used in stop names
+const stopAbbreviations = {
+    st: 'saint'
+};
+
+/**
+ * Convert a stop name to a canonical representation suitable for
+ * comparing two names.
+ *
+ * @param stopName Original stop name.
+ * @return List of normalized tokens in the name.
+ */
+const canonicalizeStopName = stopName => stopName
+    .toLowerCase()
+
+    // Remove diacritics
+    .normalize("NFD").replace(/[\u0300-\u036f]/g, "")
+
+    // Only keep alpha-numeric characters
+    .replace(/[^a-z0-9]/g, ' ')
+
+    // Split in tokens longer than two characters
+    .split(/\s+/g).filter(part => part.length >= 2)
+
+    // Resolve well-known abbreviations
+    .map(part => part in stopAbbreviations ? stopAbbreviations[part] : part);
+
+/**
+ * Compute a matching score between two stop names.
+ *
+ * @param fullName Stop name in full.
+ * @param abbrName Abbreviated stop name.
+ * @return Matching score (number of common tokens).
+ */
+const matchStopNames = (fullName, abbrName) =>
+{
+    const canonicalFullName = canonicalizeStopName(fullName);
+    const canonicalAbbrName = canonicalizeStopName(abbrName);
+
+    return canonicalFullName.filter(part =>
+        canonicalAbbrName.findIndex(abbrPart =>
+            part.startsWith(abbrPart)
+        ) !== -1
+    ).length;
+};
+
+/**
+ * Fetch stops, segments and lines in the network.
+ *
+ * @param lineRefs List of lines to fetch.
+ * @return Object with a set of stops, segments and lines.
+ */
+const fetch = async (lineRefs) =>
 {
     // Retrieve routes, ways and stops from OpenStreetMap
-    const rawData = await requestp.post(OVERPASS_ENDPOINT, {form: `\
+    const rawData = await queryOverpass(`[out:json];
-data=[out:json];
 
 // Find the public transport line bearing the requested reference
 relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"];
@@ -18,7 +135,10 @@ relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"];
 (._; >>;);
 
 out body qt;
-    `});
+`);
+
+    // Retrieve stop associations from TaM
+    const associations = await fetchStopsRefAssociations();
 
     // List of retrieved objects
     const elementsList = JSON.parse(rawData).elements;
@@ -63,8 +183,70 @@ out body qt;
 
                     if (!('ref' in stop.tags))
                     {
-                        console.warn(`Stop ${stop.id} is missing a “ref” tag`);
+                        console.warn(`Stop ${stop.id} is missing a “ref” tag
-                        continue;
+Name: ${stop.tags.name}
+Part of line: ${lineRef} (to ${route.tags.to})
+URI: ${osmViewNode}/${stop.id}
+`);
+
+                        // Try to identify stops matching this stop in the
+                        // TaM-provided data, using the stop name, line number
+                        // and trip direction
+                        const candidates = Object.entries(associations).filter(
+                            ([stopRef, {name, lines, directions}]) =>
+                            lines.has(lineRef)
+                        ).map(([stopRef, {name, lines, directions}]) => ({
+                            stopRef,
+                            lines,
+
+                            name,
+                            nameScore: matchStopNames(stop.tags.name, name),
+
+                            directions,
+                            directionScore: Math.max(
+                                ...Array.from(directions).map(direction =>
+                                    matchStopNames(route.tags.to, direction)
+                                )
+                            ),
+                        }))
+                            // Only keep non-zero scores for both criteria
+                            .filter(({nameScore, directionScore}) =>
+                                nameScore && directionScore
+                            )
+                            // Sort by best name score then best direction
+                            .sort(({
+                                nameScore: nameScore1,
+                                directionScore: directionScore1
+                            }, {
+                                nameScore: nameScore2,
+                                directionScore: directionScore2
+                            }) =>
+                                (nameScore2 - nameScore1)
+                                || (directionScore2 - directionScore1)
+                            )
+                            .slice(0, 4);
+
+                        if (candidates.length === 0)
+                        {
+                            console.warn('No candidate found in TaM data.');
+                        }
+                        else
+                        {
+                            console.warn('Candidates:');
+
+                            for (let candidate of candidates)
+                            {
+                                console.warn(`\
+    — Stop ${candidate.stopRef} with name “${candidate.name}” used by \
+${choosePlural(candidate.lines.length, 'line', '.s')} \
+${joinSentence(Array.from(candidate.lines), ', ', ' and ')} going to \
+${joinSentence(Array.from(candidate.directions), ', ', ' or ')}
+      Apply in JOSM: ${josmAddTagToNode(stop.id, ['ref=' + candidate.stopRef])}
+`);
+                            }
+                        }
+
+                        console.warn('');
                     }
 
                     if (!(stop.tags.ref in stops))
@@ -249,6 +431,6 @@ out body qt;
     }
 
     return {stops, segments, lines};
-});
+};
 
 exports.fetch = fetch;

back/data/realtime.js

@@ -2,8 +2,7 @@ const request = require('request');
 const csv = require('csv-parse');
 
 const network = require('./network');
-
+const {TAM_REALTIME} = require('./endpoints');
-const TAM_REALTIME = 'http://data.montpellier3m.fr/node/10732/download';
 
 const sortByFirstKey = (a, b) => a[0] - b[0];
 
@@ -94,12 +93,13 @@ const updateVehicles = async (lines, vehicles) =>
 
                     // Find the preceding stop from which the vehicle is coming
                     const arrivingStop = stopIndices[0][1];
-                    const arrivingStopIndex =
+                    const arrivingStopIndex = route.stops.findIndex(
-                        route.stops.findIndex(stop => stop.ref === arrivingStop);
+                        stop => stop.ref === arrivingStop
+                    );
 
-
+                    const leavingStop = arrivingStopIndex === 0
-
+                        ? route.stops[0]
-                    const [eta, nextStop] = stopIndices[0];
+                        : route.stops[arrivingStopIndex - 1];
 
                     if (nextStop === 0)
                     {

back/util.js

@@ -1,46 +1,60 @@
 /**
- * Wrap an async function to cache its result.
+ * Choose between singular or plural form based on the number of elements.
  *
- * On first call of the wrapped function, the result will be stored and further
+ * @example
- * calls will directly return this cached value (always inside a promise).
+ * > choosePlural(1, 'example', '.s')
+ * 'example'
+ * > choosePlural(4, 'example', '.s')
+ * 'examples'
+ * > choosePlural(0, 'radius', 'radii')
+ * 'radii'
  *
- * A `.noCache` method is provided to force getting a fresh value.
+ * @param count Number of elements.
- *
+ * @param singular Singular form.
- * Each cache value is scoped to the array of arguments that yielded it
+ * @param plural Plural form. An initial dot will be replaced by `singular`.
- * (comparison done using its JSON representation).
+ * @return Appropriate form.
- *
- * @param fun Function to wrap.
- * @return Wrapped function.
  */
-const makeCached = fun =>
+const choosePlural = (count, singular, plural) =>
 {
-    const cachedResults = new Map();
+    if (count === 1)
-
-    const noCache = async (...args) =>
     {
-        const result = await fun(...args);
+        return singular;
-        cachedResults.set(JSON.stringify(args), result);
+    }
-        return result;
+    else
-    };
-
-    const withCache = async (...args) =>
     {
-        const key = JSON.stringify(args);
+        return plural.replace(/^\./, singular);
-
+    }
-        if (cachedResults.has(key))
-        {
-            return cachedResults.get(key);
-        }
-        else
-        {
-            const result = await fun(...args);
-            cachedResults.set(key, result);
-            return result;
-        }
-    };
-
-    withCache.noCache = noCache;
-    return withCache;
 };
 
-exports.makeCached = makeCached;
+exports.choosePlural = choosePlural;
+
+/**
+ * Join elements with the given separator and a special separator for the last
+ * element.
+ *
+ * @example
+ * > joinSentence(['apple', 'orange', 'banana'], ', ', ' and ')
+ * 'apple, orange and banana'
+ * > joinSentence(['apple', 'banana'], ', ', ' and ')
+ * 'apple and banana'
+ * > joinSentence(['banana'], ', ', ' and ')
+ * 'banana'
+ *
+ * @param array Sequence of strings to join.
+ * @param separator Separator for all elements but the last one.
+ * @param lastSeparator Separator for the last element.
+ * @return Joined string.
+ */
+const joinSentence = (array, separator, lastSeparator) =>
+{
+    if (array.length <= 2)
+    {
+        return array.join(lastSeparator);
+    }
+
+    return array.slice(0, -1).join(separator)
+        + lastSeparator
+        + array[array.length - 1];
+};
+
+exports.joinSentence = joinSentence;

front/index.js

@@ -4,7 +4,10 @@ const color = require('color');
 require('leaflet/dist/leaflet.css');
 
 // MAP
-const map = L.map('map').setView([43.610, 3.8612], 14);
+const map = L.map('map').setView(
+    [43.605, 3.88],
+    /* zoomLevel = */ 13
+);
 
 L.tileLayer('https://api.mapbox.com/styles/v1/{id}/tiles/{z}/{x}/{y}?access_token={accessToken}', {
     attribution: [
@@ -20,7 +23,7 @@ licenses/by-sa/2.0/">CC-BY-SA</a>`,
 
     accessToken: 'pk.eyJ1IjoibWF0dGVvZGVsYWJyZSIsImEiOiJjazUxaWNsdXcwdWhjM29tc2xndXJoNGtxIn0.xELwMerqJLFimIqU6RxnZw',
 
-    maxZoom: 18,
+    maxZoom: 19,
     zoomSnap: 0,
     zoomOffset: -1,
 }).addTo(map);
@@ -52,7 +55,7 @@ fetch(SERVER + '/network').then(res => res.json()).then(network =>
         line.bindPopup(`${segment.length} m`);
     });
 
-    Object.values(network.stops).forEach(stop =>
+    Object.entries(network.stops).forEach(([stopId, stop]) =>
     {
         const color = network.lines[stop.lines[0]].color;
         const borderColor = makeBorderColor(color);
@@ -68,7 +71,8 @@ fetch(SERVER + '/network').then(res => res.json()).then(network =>
             }
         ).addTo(map);
 
-        stopMarker.bindPopup(stop.name);
+        stopMarker.bindPopup(`<strong>${stop.name}</strong><br>
+Arrêt n°${stopId}`);
     });
 
     console.log(network);

package-lock.json

@@ -1933,6 +1933,15 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "binary": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
+      "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
+      "requires": {
+        "buffers": "~0.1.1",
+        "chainsaw": "~0.1.0"
+      }
+    },
     "binary-extensions": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
@@ -2135,6 +2144,11 @@
       "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
       "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk="
     },
+    "buffers": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
+      "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
+    },
     "builtin-status-codes": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
@@ -2220,6 +2234,14 @@
       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
       "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
     },
+    "chainsaw": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
+      "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
+      "requires": {
+        "traverse": ">=0.3.0 <0.4"
+      }
+    },
     "chalk": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -6967,6 +6989,11 @@
         "punycode": "^2.1.0"
       }
     },
+    "traverse": {
+      "version": "0.3.9",
+      "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
+      "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
+    },
     "tty-browserify": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz",
@@ -7169,6 +7196,15 @@
       "integrity": "sha1-0vD3N9FrBhXnKmk17QQhRXLVb5c=",
       "dev": true
     },
+    "unzip-stream": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.0.tgz",
+      "integrity": "sha512-NG1h/MdGIX3HzyqMjyj1laBCmlPYhcO4xEy7gEqqzGiSLw7XqDQCnY4nYSn5XSaH8mQ6TFkaujrO8d/PIZN85A==",
+      "requires": {
+        "binary": "^0.3.0",
+        "mkdirp": "^0.5.1"
+      }
+    },
     "upath": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz",

package.json

@@ -18,7 +18,8 @@
     "leaflet": "^1.6.0",
     "parcel-bundler": "^1.12.4",
     "request": "^2.88.0",
-    "request-promise-native": "^1.0.8"
+    "request-promise-native": "^1.0.8",
+    "unzip-stream": "^0.3.0"
   },
   "devDependencies": {
     "nodemon": "^2.0.2"

script/update-network.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const network = require('../back/data/network');
+const path = require('path');
+const fs = require('fs');
+
+(async () =>
+{
+    const lines = ['1', '2', '3'];
+    const data = await network.fetch(lines);
+
+    fs.writeFileSync(
+        path.join(__dirname, '../back/data/network.json'),
+        JSON.stringify(data)
+    );
+})();

server.js

@@ -1,6 +1,6 @@
 const express = require('express');
 
-const network = require('./back/data/network');
+const network = require('./back/data/network.json');
 
 const app = express();
 const port = 3000;
@@ -11,10 +11,6 @@ app.get('/realtime', (req, res) =>
 {
 });
 
-app.get('/network', async (req, res) =>
+app.get('/network', async (req, res) => res.json(network));
-{
-    const networkData = await network.fetch(['1', '2', '3', '4']);
-    res.json(networkData);
-});
 
 app.listen(port, () => console.log(`Example app listening on port ${port}!`))