Store network inside a cached file

Create script for updating the network, suggest suitable refs for stops
that lack one.
This commit is contained in:
Mattéo Delabre 2020-01-15 00:19:26 +01:00
parent a4e1ee199b
commit 3c3f446503
Signed by: matteo
GPG Key ID: AE3FBD02DC583ABB
10 changed files with 430 additions and 65 deletions

115
back/data/endpoints.js Normal file
View File

@ -0,0 +1,115 @@
const unzip = require('unzip-stream');
const csv = require('csv-parse');
const request = require('request');
const requestp = require('request-promise-native');
const overpassEndpoint = 'https://lz4.overpass-api.de/api/interpreter';
/**
* Submit an Overpass query.
*
* @async
* @param query Query in Overpass QL.
* @return Results as provided by the endpoint.
*/
const queryOverpass = query => requestp.post(
overpassEndpoint,
{form: 'data=' + query}
);
exports.queryOverpass = queryOverpass;
/**
* Process a CSV stream to extract passings.
*
* @private
* @param csvStream Stream containing CSV data.
* @param callback See fetchTamRealtime for a description of the callback.
*/
const processTamPassingStream = (csvStream, callback) =>
{
const parser = csv({
delimiter: ';',
});
const rowStream = csvStream.pipe(parser);
rowStream.on('readable', () =>
{
let row;
while (row = rowStream.read())
{
if (row.length === 0 || row[0] === 'course')
{
// Ignore les lignes invalides et len-tête
continue;
}
callback(null, {
course: row[0],
stopCode: row[1],
stopId: row[2],
stopName: row[3],
routeShortName: row[4],
tripHeadsign: row[5],
directionId: row[6],
departureTime: row[7],
isTheorical: row[8],
delaySec: row[9],
destArCode: row[10],
});
}
});
rowStream.on('end', () => callback(null, null));
rowStream.on('error', err => callback(err));
};
const tamRealtimeEndpoint = 'http://data.montpellier3m.fr/node/10732/download';
/**
* Fetch realtime passings for the current day across the network.
*
* @param callback Called for each passing during parsing. First argument will
* be non-null only if an error occurred. Second argument will contain passings
* or be null if the end was reached.
*/
const fetchTamRealtime = (callback) =>
{
const csvStream = request(tamRealtimeEndpoint);
processTamPassingStream(csvStream, callback);
};
exports.fetchTamRealtime = fetchTamRealtime;
const tamTheoreticalEndpoint =
'http://data.montpellier3m.fr/node/10731/download';
const tamTheoreticalFileName = 'offre_du_jour.csv';
/**
* Fetch theoretical passings for the current day across the network.
*
* @param callback Called for each passing during parsing. First argument will
* be non-null only if an error occurred. Second argument will contain passings
* or be null if the end was reached.
*/
const fetchTamTheoretical = (callback) =>
{
const fileStream = request(tamTheoreticalEndpoint).pipe(unzip.Parse());
fileStream.on('entry', entry =>
{
if (entry.type !== 'File' || entry.path !== tamTheoreticalFileName)
{
entry.autodrain();
return;
}
processTamPassingStream(entry, callback);
});
fileStream.on('error', err => callback(err));
};
exports.fetchTamTheoretical = fetchTamTheoretical;

View File

@ -1,15 +1,132 @@
const requestp = require('request-promise-native');
const geolib = require('geolib');
const {makeCached} = require('../util');
const {choosePlural, joinSentence} = require('../util');
const {queryOverpass, fetchTamTheoretical} = require('./endpoints');
const OVERPASS_ENDPOINT = 'https://lz4.overpass-api.de/api/interpreter';
const osmViewNode = 'https://www.openstreetmap.org/node';
const fetch = makeCached(async (lineRefs) =>
/**
* Create a link to remotely add tags into JOSM.
*
* @param objectId Identifier for the object to add the tags to.
* @param tags Tags to add.
* @return Link for remotely adding the tags.
*/
const josmAddTagToNode = (objectId, tags) =>
'http://127.0.0.1:8111/load_object?' + [
`objects=n${objectId}`,
'new_layer=false',
'addtags=' + tags.join('%7C')
].join('&');
/**
* Use theoretical passings data to guess which lines use which stops in which
* direction.
*
* This is used for suggesting possible stop IDs for stops that dont have
* one in OSM.
*
* @return Map containing for each stop its abbreviated name, the lines that
* use it and in which directions it is used.
*/
const fetchStopsRefAssociations = () => new Promise((res, rej) =>
{
const stops = {};
fetchTamTheoretical((err, row) =>
{
if (err)
{
rej(err);
return;
}
if (!row)
{
res(stops);
return;
}
if (!(row.stopId in stops))
{
stops[row.stopId] = {
name: row.stopName,
lines: new Set([row.routeShortName]),
directions: new Set([row.tripHeadsign]),
};
}
else
{
const stop = stops[row.stopId];
if (stop.name !== row.stopName)
{
console.warn(`Stop ${row.stopId} has multiple names: \
${row.stopName} and ${stop.name}. Only the first one will be considered.`);
}
stop.lines.add(row.routeShortName);
stop.directions.add(row.tripHeadsign);
}
});
});
// Mapping for abbreviations used in stop names
const stopAbbreviations = {
st: 'saint'
};
/**
* Convert a stop name to a canonical representation suitable for
* comparing two names.
*
* @param stopName Original stop name.
* @return List of normalized tokens in the name.
*/
const canonicalizeStopName = stopName => stopName
.toLowerCase()
// Remove diacritics
.normalize("NFD").replace(/[\u0300-\u036f]/g, "")
// Only keep alpha-numeric characters
.replace(/[^a-z0-9]/g, ' ')
// Split in tokens longer than two characters
.split(/\s+/g).filter(part => part.length >= 2)
// Resolve well-known abbreviations
.map(part => part in stopAbbreviations ? stopAbbreviations[part] : part);
/**
* Compute a matching score between two stop names.
*
* @param fullName Stop name in full.
* @param abbrName Abbreviated stop name.
* @return Matching score (number of common tokens).
*/
const matchStopNames = (fullName, abbrName) =>
{
const canonicalFullName = canonicalizeStopName(fullName);
const canonicalAbbrName = canonicalizeStopName(abbrName);
return canonicalFullName.filter(part =>
canonicalAbbrName.findIndex(abbrPart =>
part.startsWith(abbrPart)
) !== -1
).length;
};
/**
* Fetch stops, segments and lines in the network.
*
* @param lineRefs List of lines to fetch.
* @return Object with a set of stops, segments and lines.
*/
const fetch = async (lineRefs) =>
{
// Retrieve routes, ways and stops from OpenStreetMap
const rawData = await requestp.post(OVERPASS_ENDPOINT, {form: `\
data=[out:json];
const rawData = await queryOverpass(`[out:json];
// Find the public transport line bearing the requested reference
relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"];
@ -18,7 +135,10 @@ relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"];
(._; >>;);
out body qt;
`});
`);
// Retrieve stop associations from TaM
const associations = await fetchStopsRefAssociations();
// List of retrieved objects
const elementsList = JSON.parse(rawData).elements;
@ -63,8 +183,70 @@ out body qt;
if (!('ref' in stop.tags))
{
console.warn(`Stop ${stop.id} is missing a “ref” tag`);
continue;
console.warn(`Stop ${stop.id} is missing a “ref” tag
Name: ${stop.tags.name}
Part of line: ${lineRef} (to ${route.tags.to})
URI: ${osmViewNode}/${stop.id}
`);
// Try to identify stops matching this stop in the
// TaM-provided data, using the stop name, line number
// and trip direction
const candidates = Object.entries(associations).filter(
([stopRef, {name, lines, directions}]) =>
lines.has(lineRef)
).map(([stopRef, {name, lines, directions}]) => ({
stopRef,
lines,
name,
nameScore: matchStopNames(stop.tags.name, name),
directions,
directionScore: Math.max(
...Array.from(directions).map(direction =>
matchStopNames(route.tags.to, direction)
)
),
}))
// Only keep non-zero scores for both criteria
.filter(({nameScore, directionScore}) =>
nameScore && directionScore
)
// Sort by best name score then best direction
.sort(({
nameScore: nameScore1,
directionScore: directionScore1
}, {
nameScore: nameScore2,
directionScore: directionScore2
}) =>
(nameScore2 - nameScore1)
|| (directionScore2 - directionScore1)
)
.slice(0, 4);
if (candidates.length === 0)
{
console.warn('No candidate found in TaMdata.');
}
else
{
console.warn('Candidates:');
for (let candidate of candidates)
{
console.warn(`\
Stop ${candidate.stopRef} with name ${candidate.name} used by \
${choosePlural(candidate.lines.length, 'line', '.s')} \
${joinSentence(Array.from(candidate.lines), ', ', ' and ')} going to \
${joinSentence(Array.from(candidate.directions), ', ', ' or ')}
Apply in JOSM: ${josmAddTagToNode(stop.id, ['ref=' + candidate.stopRef])}
`);
}
}
console.warn('');
}
if (!(stop.tags.ref in stops))
@ -249,6 +431,6 @@ out body qt;
}
return {stops, segments, lines};
});
};
exports.fetch = fetch;

1
back/data/network.json Normal file

File diff suppressed because one or more lines are too long

View File

@ -2,8 +2,7 @@ const request = require('request');
const csv = require('csv-parse');
const network = require('./network');
const TAM_REALTIME = 'http://data.montpellier3m.fr/node/10732/download';
const {TAM_REALTIME} = require('./endpoints');
const sortByFirstKey = (a, b) => a[0] - b[0];
@ -94,12 +93,13 @@ const updateVehicles = async (lines, vehicles) =>
// Find the preceding stop from which the vehicle is coming
const arrivingStop = stopIndices[0][1];
const arrivingStopIndex =
route.stops.findIndex(stop => stop.ref === arrivingStop);
const arrivingStopIndex = route.stops.findIndex(
stop => stop.ref === arrivingStop
);
const [eta, nextStop] = stopIndices[0];
const leavingStop = arrivingStopIndex === 0
? route.stops[0]
: route.stops[arrivingStopIndex - 1];
if (nextStop === 0)
{

View File

@ -1,46 +1,60 @@
/**
* Wrap an async function to cache its result.
* Choose between singular or plural form based on the number of elements.
*
* On first call of the wrapped function, the result will be stored and further
* calls will directly return this cached value (always inside a promise).
* @example
* > choosePlural(1, 'example', '.s')
* 'example'
* > choosePlural(4, 'example', '.s')
* 'examples'
* > choosePlural(0, 'radius', 'radii')
* 'radii'
*
* A `.noCache` method is provided to force getting a fresh value.
*
* Each cache value is scoped to the array of arguments that yielded it
* (comparison done using its JSON representation).
*
* @param fun Function to wrap.
* @return Wrapped function.
* @param count Number of elements.
* @param singular Singular form.
* @param plural Plural form. An initial dot will be replaced by `singular`.
* @return Appropriate form.
*/
const makeCached = fun =>
const choosePlural = (count, singular, plural) =>
{
const cachedResults = new Map();
const noCache = async (...args) =>
if (count === 1)
{
const result = await fun(...args);
cachedResults.set(JSON.stringify(args), result);
return result;
};
const withCache = async (...args) =>
return singular;
}
else
{
const key = JSON.stringify(args);
if (cachedResults.has(key))
{
return cachedResults.get(key);
}
else
{
const result = await fun(...args);
cachedResults.set(key, result);
return result;
}
};
withCache.noCache = noCache;
return withCache;
return plural.replace(/^\./, singular);
}
};
exports.makeCached = makeCached;
exports.choosePlural = choosePlural;
/**
* Join elements with the given separator and a special separator for the last
* element.
*
* @example
* > joinSentence(['apple', 'orange', 'banana'], ', ', ' and ')
* 'apple, orange and banana'
* > joinSentence(['apple', 'banana'], ', ', ' and ')
* 'apple and banana'
* > joinSentence(['banana'], ', ', ' and ')
* 'banana'
*
* @param array Sequence of strings to join.
* @param separator Separator for all elements but the last one.
* @param lastSeparator Separator for the last element.
* @return Joined string.
*/
const joinSentence = (array, separator, lastSeparator) =>
{
if (array.length <= 2)
{
return array.join(lastSeparator);
}
return array.slice(0, -1).join(separator)
+ lastSeparator
+ array[array.length - 1];
};
exports.joinSentence = joinSentence;

View File

@ -4,7 +4,10 @@ const color = require('color');
require('leaflet/dist/leaflet.css');
// MAP
const map = L.map('map').setView([43.610, 3.8612], 14);
const map = L.map('map').setView(
[43.605, 3.88],
/* zoomLevel = */ 13
);
L.tileLayer('https://api.mapbox.com/styles/v1/{id}/tiles/{z}/{x}/{y}?access_token={accessToken}', {
attribution: [
@ -20,7 +23,7 @@ licenses/by-sa/2.0/">CC-BY-SA</a>`,
accessToken: 'pk.eyJ1IjoibWF0dGVvZGVsYWJyZSIsImEiOiJjazUxaWNsdXcwdWhjM29tc2xndXJoNGtxIn0.xELwMerqJLFimIqU6RxnZw',
maxZoom: 18,
maxZoom: 19,
zoomSnap: 0,
zoomOffset: -1,
}).addTo(map);
@ -52,7 +55,7 @@ fetch(SERVER + '/network').then(res => res.json()).then(network =>
line.bindPopup(`${segment.length} m`);
});
Object.values(network.stops).forEach(stop =>
Object.entries(network.stops).forEach(([stopId, stop]) =>
{
const color = network.lines[stop.lines[0]].color;
const borderColor = makeBorderColor(color);
@ -68,7 +71,8 @@ fetch(SERVER + '/network').then(res => res.json()).then(network =>
}
).addTo(map);
stopMarker.bindPopup(stop.name);
stopMarker.bindPopup(`<strong>${stop.name}</strong><br>
Arrêt n°${stopId}`);
});
console.log(network);

36
package-lock.json generated
View File

@ -1933,6 +1933,15 @@
"tweetnacl": "^0.14.3"
}
},
"binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
"requires": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
}
},
"binary-extensions": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
@ -2135,6 +2144,11 @@
"resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
"integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk="
},
"buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
},
"builtin-status-codes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
@ -2220,6 +2234,14 @@
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
"requires": {
"traverse": ">=0.3.0 <0.4"
}
},
"chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@ -6967,6 +6989,11 @@
"punycode": "^2.1.0"
}
},
"traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
},
"tty-browserify": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz",
@ -7169,6 +7196,15 @@
"integrity": "sha1-0vD3N9FrBhXnKmk17QQhRXLVb5c=",
"dev": true
},
"unzip-stream": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.0.tgz",
"integrity": "sha512-NG1h/MdGIX3HzyqMjyj1laBCmlPYhcO4xEy7gEqqzGiSLw7XqDQCnY4nYSn5XSaH8mQ6TFkaujrO8d/PIZN85A==",
"requires": {
"binary": "^0.3.0",
"mkdirp": "^0.5.1"
}
},
"upath": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz",

View File

@ -18,7 +18,8 @@
"leaflet": "^1.6.0",
"parcel-bundler": "^1.12.4",
"request": "^2.88.0",
"request-promise-native": "^1.0.8"
"request-promise-native": "^1.0.8",
"unzip-stream": "^0.3.0"
},
"devDependencies": {
"nodemon": "^2.0.2"

16
script/update-network.js Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env node
const network = require('../back/data/network');
const path = require('path');
const fs = require('fs');
(async () =>
{
const lines = ['1', '2', '3'];
const data = await network.fetch(lines);
fs.writeFileSync(
path.join(__dirname, '../back/data/network.json'),
JSON.stringify(data)
);
})();

View File

@ -1,6 +1,6 @@
const express = require('express');
const network = require('./back/data/network');
const network = require('./back/data/network.json');
const app = express();
const port = 3000;
@ -11,10 +11,6 @@ app.get('/realtime', (req, res) =>
{
});
app.get('/network', async (req, res) =>
{
const networkData = await network.fetch(['1', '2', '3', '4']);
res.json(networkData);
});
app.get('/network', async (req, res) => res.json(network));
app.listen(port, () => console.log(`Example app listening on port ${port}!`))