Retrieve whole network and start of realtime

This commit is contained in:
Mattéo Delabre 2020-01-14 14:08:08 +01:00
parent bbc1e89aa9
commit a4e1ee199b
Signed by: matteo
GPG Key ID: AE3FBD02DC583ABB
7 changed files with 464 additions and 284 deletions

View File

@ -1,198 +0,0 @@
const requestp = require('request-promise-native');
const {makeCached} = require('./util');
const OVERPASS_ENDPOINT = 'https://lz4.overpass-api.de/api/interpreter';
const fetchLineData = makeCached(async (lineRef) =>
{
// Retrieve routes, ways and stops from OpenStreetMap
const rawData = await requestp.post(OVERPASS_ENDPOINT, {form: `\
data=[out:json];
// Find the public transport line bearing the requested reference
relation[network="TaM"][type="route_master"][ref="${lineRef}"];
// Recursively fetch routes, ways and stops inside the line
(._; >>;);
out body qt;
`});
const elementsList = JSON.parse(rawData).elements;
// Extract all routes for the given line
const rawRoutes = elementsList.filter(elt =>
elt.type === 'relation'
&& elt.tags.type === 'route'
&& elt.tags.ref === lineRef
);
// If no route is found, assume the line does not exist
if (rawRoutes.length === 0)
{
return null;
}
// Index retrieved objects by their ID
const elements = elementsList.reduce((prev, elt) =>
{
prev[elt.id] = elt;
return prev;
}, {});
const color = rawRoutes[0].tags.colour || '#000000';
// Extract stops for each route of the line
const routes = rawRoutes.map(route => ({
from: route.tags.from,
to: route.tags.to,
// Retrieve each stops information (stop order in the relation is
// assumed to reflect reality)
stops: route.members
.filter(({role}) => role === 'stop')
.map(({ref}) => {
const elt = elements[ref];
return {
id: ref,
lat: elt.lat,
lon: elt.lon,
name: elt.tags.name,
};
}),
// List of ways making up the route (raw)
rawWays: route.members
.filter(({role}) => role === '')
.map(({ref}) => ref)
}));
// Process ways in each route to sort and merge them
for (let route of routes)
{
const {from, to, stops, rawWays} = route;
// Construct a graph with all connected nodes
const nodeNeighbors = new Map();
for (let id of rawWays)
{
const {type, nodes, tags} = elements[id];
const isOneWay = (
tags.oneway === 'yes'
|| tags.junction === 'roundabout'
);
const canGoBackward = (
!isOneWay
|| parseInt(tags['lanes:psv:backward'], 10) > 0
);
if (type === 'way' && nodes.length >= 1)
{
let previousNode = nodes[0];
if (!nodeNeighbors.has(previousNode))
{
nodeNeighbors.set(previousNode, new Set());
}
for (let node of nodes.slice(1))
{
if (!nodeNeighbors.has(node))
{
nodeNeighbors.set(node, new Set());
}
nodeNeighbors.get(previousNode).add(node);
if (canGoBackward)
{
nodeNeighbors.get(node).add(previousNode);
}
previousNode = node;
}
}
}
// Find way from first stop through the end using DFS
const numberOfStops = stops.length;
const ways = [];
let currentStopIndex = 0;
while (currentStopIndex + 1 < numberOfStops)
{
const currentStop = stops[currentStopIndex];
const nextStop = stops[currentStopIndex + 1];
const visitedEdges = new Set();
const stack = [{
currentNode: currentStop.id,
way: [currentStop.id],
}];
let found = false;
while (stack.length !== 0)
{
const {currentNode, way} = stack.pop();
if (currentNode === nextStop.id)
{
// Arrived at next stop
ways.push(way);
found = true;
break;
}
const neighbors = nodeNeighbors.get(currentNode) || [];
for (let nextNode of neighbors)
{
const edge = `${currentNode}-${nextNode}`;
if (!visitedEdges.has(edge))
{
visitedEdges.add(edge);
stack.push({
currentNode: nextNode,
way: way.concat([nextNode]),
});
}
}
}
if (!found)
{
throw new Error(`No way between stop “${currentStop.name}\
(${currentStop.id}) and stop ${nextStop.name} (${nextStop.id}) on line \
${lineRef}s route from ${from} to ${to}`);
}
++currentStopIndex;
}
// Only keep geo coordinates for each way node
route.ways = ways.map(way =>
way.map(id =>
{
const node = elements[id];
return {lat: node.lat, lon: node.lon};
})
);
delete route.rawWays;
}
return {
ref: lineRef,
color,
routes
};
});
exports.fetchLineData = fetchLineData;

254
back/data/network.js Normal file
View File

@ -0,0 +1,254 @@
const requestp = require('request-promise-native');
const geolib = require('geolib');
const {makeCached} = require('../util');
const OVERPASS_ENDPOINT = 'https://lz4.overpass-api.de/api/interpreter';
const fetch = makeCached(async (lineRefs) =>
{
// Retrieve routes, ways and stops from OpenStreetMap
const rawData = await requestp.post(OVERPASS_ENDPOINT, {form: `\
data=[out:json];
// Find the public transport line bearing the requested reference
relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"];
// Recursively fetch routes, ways and stops inside the line
(._; >>;);
out body qt;
`});
// List of retrieved objects
const elementsList = JSON.parse(rawData).elements;
// List of retrieved lines
const routeMasters = elementsList.filter(elt =>
elt.tags && elt.tags.type === 'route_master'
);
// Retrieved objects indexed by ID
const elements = elementsList.reduce((prev, elt) =>
{
prev[elt.id] = elt;
return prev;
}, {});
// Result object containing all stops
const stops = {};
// Result object containing all segments between stops
const segments = {};
// Result object containing all lines
const lines = {};
for (let routeMaster of routeMasters)
{
const lineRef = routeMaster.tags.ref;
const color = routeMaster.tags.colour || '#000000';
// Extract all routes for the given line
const rawRoutes = routeMaster.members.map(({ref}) => elements[ref]);
// Add missing stops to the result object
for (let route of rawRoutes)
{
for (let {ref, role} of route.members)
{
if (role === 'stop')
{
const stop = elements[ref];
if (!('ref' in stop.tags))
{
console.warn(`Stop ${stop.id} is missing a “ref” tag`);
continue;
}
if (!(stop.tags.ref in stops))
{
stops[stop.tags.ref] = {
lat: stop.lat,
lon: stop.lon,
name: stop.tags.name,
lines: [lineRef],
};
}
else
{
stops[stop.tags.ref].lines.push(lineRef);
}
}
}
}
// Add missing segments between stops
for (let route of rawRoutes)
{
const {from, to} = route.tags;
const stops = route.members
.filter(({role}) => role === 'stop')
.map(({ref}) => elements[ref])
.filter(stop => 'ref' in stop.tags)
.map(stop => ({
id: stop.id,
lat: stop.lat,
lon: stop.lon,
ref: stop.tags.ref,
name: stop.tags.name,
}));
const ways = route.members
.filter(({role}) => role === '')
.map(({ref}) => ref);
// Construct a graph with all connected nodes
const nodeNeighbors = new Map();
for (let id of ways)
{
const {type, nodes, tags} = elements[id];
const isOneWay = (
tags.oneway === 'yes'
|| tags.junction === 'roundabout'
);
const canGoBackward = (
!isOneWay
|| parseInt(tags['lanes:psv:backward'], 10) > 0
);
if (type === 'way' && nodes.length >= 1)
{
let previousNode = nodes[0];
if (!nodeNeighbors.has(previousNode))
{
nodeNeighbors.set(previousNode, new Set());
}
for (let node of nodes.slice(1))
{
if (!nodeNeighbors.has(node))
{
nodeNeighbors.set(node, new Set());
}
nodeNeighbors.get(previousNode).add(node);
if (canGoBackward)
{
nodeNeighbors.get(node).add(previousNode);
}
previousNode = node;
}
}
}
// Find way from first stop through the end using DFS
const numberOfStops = stops.length;
let currentStopIndex = 0;
while (currentStopIndex + 1 < numberOfStops)
{
const currentStop = stops[currentStopIndex];
const nextStop = stops[currentStopIndex + 1];
const segmentId = `${currentStop.ref}-${nextStop.ref}`;
if (!(segmentId in segments))
{
const visitedEdges = new Set();
const stack = [{
currentNode: currentStop.id,
segment: [currentStop.id],
}];
let found = false;
while (stack.length !== 0)
{
const {currentNode, segment} = stack.pop();
if (currentNode === nextStop.id)
{
// Arrived at next stop
segments[segmentId] = {
nodes: segment.map(id =>
{
const {lat, lon} = elements[id];
return {lat, lon};
}),
length: geolib.getPathLength(segment.map(id =>
{
const {lat, lon} = elements[id];
return {latitude: lat, longitude: lon};
})),
lines: [lineRef],
};
found = true;
break;
}
const neighbors = nodeNeighbors.get(currentNode) || [];
for (let nextNode of neighbors)
{
const edge = `${currentNode}-${nextNode}`;
if (!visitedEdges.has(edge))
{
visitedEdges.add(edge);
stack.push({
currentNode: nextNode,
segment: segment.concat([nextNode]),
});
}
}
}
if (!found)
{
throw new Error(`No way between stop \
${currentStop.name} (${currentStop.id}) and stop ${nextStop.name} \
(${nextStop.id}) on line ${lineRef}s route from ${from} to ${to}`);
}
}
else
{
segments[segmentId].lines.push(lineRef);
}
++currentStopIndex;
}
}
// Construct line objects
const routes = rawRoutes.map(route => ({
from: route.tags.from,
to: route.tags.to,
// Retrieve each stops information (stop order in the relation is
// assumed to reflect reality)
stops: route.members
.filter(({role}) => role === 'stop')
.map(({ref}) => elements[ref])
.filter(stop => 'ref' in stop.tags)
.map(stop => stop.tags.ref)
}));
lines[lineRef] = {
color,
routes
};
}
return {stops, segments, lines};
});
exports.fetch = fetch;

172
back/data/realtime.js Normal file
View File

@ -0,0 +1,172 @@
const request = require('request');
const csv = require('csv-parse');
const network = require('./network');
const TAM_REALTIME = 'http://data.montpellier3m.fr/node/10732/download';
const sortByFirstKey = (a, b) => a[0] - b[0];
const fetchRealtime = () => new Promise((res, rej) =>
{
const parser = csv({
delimiter: ';',
});
const stream = request(TAM_REALTIME).pipe(parser);
const courses = {};
stream.on('readable', () =>
{
let row;
while (row = stream.read())
{
if (row.length === 0 || row[0] === 'course')
{
// Ignore les lignes invalides et len-tête
continue;
}
const course = row[0];
const stopRef = row[2];
const lineRef = row[4];
const eta = row[9];
const destinationRef = row[10];
if (!(course in courses))
{
courses[course] = {
lineRef,
destinationRef,
stops: [],
};
}
courses[course].stops.push([parseInt(eta, 10), stopRef]);
courses[course].stops.sort(sortByFirstKey);
}
});
stream.on('end', () => res(courses));
stream.on('error', err => rej(err));
});
const updateVehicles = async (lines, vehicles) =>
{
const courses = await fetchRealtime();
const currentTime = Math.floor(Date.now() / 1000);
for (let [courseRef, course] of Object.entries(courses))
{
if (course.lineRef in lines)
{
if (!(courseRef in vehicles))
{
// New vehicle: identify which route it pertains to
const line = lines[course.lineRef];
let routeIndex = null;
for (let [index, route] of Object.entries(line.routes))
{
const destRef = route.stops[route.stops.length - 1].ref;
if (destRef === course.destinationRef)
{
routeIndex = index;
}
}
if (routeIndex !== null)
{
const route = line.routes[routeIndex];
// Convert ETAs to absolute times
const nextStops = course.stops.map(([eta, ref]) => [
eta + currentTime,
ref
]);
// Convert stop refs to indices
const stopIndices = course.stops.map(([eta, ref]) => [
eta,
]);
// Find the preceding stop from which the vehicle is coming
const arrivingStop = stopIndices[0][1];
const arrivingStopIndex =
route.stops.findIndex(stop => stop.ref === arrivingStop);
const [eta, nextStop] = stopIndices[0];
if (nextStop === 0)
{
// Vehicle at starting point
vehicles[courseRef] = {
lineRef: course.lineRef,
stopRef
stopIndex: 0,
nextStops: stopIndices,
distance: 0,
speed: 0,
};
}
else
{
// Vehicle in transit between two stops
vehicles[courseRef] = {
lineRef: course.lineRef,
routeIndex,
stopIndex: nextStop - 1,
nextStops: stopIndices,
distance: 0,
speed: route.distances[nextStop - 1] / eta,
};
}
}
}
else
{
// Existing vehicle: update information
const vehicle = vehicles[courseRef];
const line = lines[vehicle.lineRef];
const route = line.routes[vehicle.routeIndex];
// Convert stop refs to indices
const stopIndices = course.stops.map(([eta, ref]) => [
eta,
route.stops.findIndex(stop => stop.ref === ref),
]);
console.log(stopIndices);
console.log(vehicle);
console.log(course);
console.log('---');
}
}
}
};
const sleep = time => new Promise(res => setTimeout(res, time));
const updateLoop = async (lines, vehicles = {}) =>
{
await updateVehicles(lines, vehicles);
await sleep(30000);
return updateLoop(lines, vehicles);
};
(async () =>
{
const lines = {'1': await network.fetchLineData('1')};
updateLoop(lines);
// console.log(require('util').inspect(vehicles, true, 10));
})();

View File

@ -32,41 +32,31 @@ licenses/by-sa/2.0/">CC-BY-SA</a>`,
const makeBorderColor = mainColor =>
{
const hsl = color(mainColor).hsl();
if (hsl.color[2] < 40)
{
hsl.color[2] += 30;
}
else
{
hsl.color[2] -= 20;
}
hsl.color = Math.max(0, hsl.color[2] -= 20);
return hsl.hex();
};
const SERVER = window.origin;
fetch(SERVER + '/line/1').then(res => res.json()).then(line =>
fetch(SERVER + '/network').then(res => res.json()).then(network =>
{
const color = line.color;
Object.values(network.segments).forEach(segment =>
{
const color = network.lines[segment.lines[0]].color;
const borderColor = makeBorderColor(color);
const nodes = segment.nodes.map(({lat, lon}) => [lat, lon]);
L.polyline(nodes, {weight: 8, color: borderColor}).addTo(map);
const line = L.polyline(nodes, {weight: 6, color}).addTo(map);
line.bindPopup(`${segment.length} m`);
});
Object.values(network.stops).forEach(stop =>
{
const color = network.lines[stop.lines[0]].color;
const borderColor = makeBorderColor(color);
line.routes.forEach(route =>
{
route.ways.forEach(way =>
{
const wayPoints = way.map(({lat, lon}) => [lat, lon]);
L.polyline(wayPoints, {weight: 8, color: borderColor}).addTo(map);
L.polyline(wayPoints, {weight: 6, color}).addTo(map);
});
});
line.routes.forEach(route =>
{
route.stops.forEach(stop =>
{
const stopMarker = L.circleMarker(
[stop.lat, stop.lon],
{
@ -80,5 +70,6 @@ fetch(SERVER + '/line/1').then(res => res.json()).then(line =>
stopMarker.bindPopup(stop.name);
});
});
console.log(network);
});

5
package-lock.json generated
View File

@ -3616,6 +3616,11 @@
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz",
"integrity": "sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg=="
},
"geolib": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/geolib/-/geolib-3.2.1.tgz",
"integrity": "sha512-O9nD8iSD4VimupKak8bKySLkkWI5VWetxIXsU7jmJRXxBFRR9LxSXGfTomtcHJLSRiznx+YHXHTOIVq4qgQmPw=="
},
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",

View File

@ -14,6 +14,7 @@
"color": "^3.1.2",
"csv-parse": "^4.8.3",
"express": "^4.17.1",
"geolib": "^3.2.1",
"leaflet": "^1.6.0",
"parcel-bundler": "^1.12.4",
"request": "^2.88.0",

View File

@ -1,65 +1,20 @@
const express = require('express');
const request = require('request');
const csv = require('csv-parse');
const data = require('./back/data');
const network = require('./back/data/network');
const app = express();
const port = 3000;
app.use(express.static('dist'));
const TAM_REALTIME = 'http://data.montpellier3m.fr/node/10732/download';
app.get('/realtime', (req, res) =>
{
const parser = csv({
delimiter: ';',
});
passages = {};
passagesLast = Date.now();
const stream = request(TAM_ENDPOINT)
.pipe(parser);
stream.on('readable', () =>
app.get('/network', async (req, res) =>
{
let row;
while (row = stream.read())
{
if (row.length === 0 || row[0] === 'course')
{
// Ignore les lignes invalides
continue;
}
if (passages[row[2]] === undefined)
{
passages[row[2]] = [];
}
passages[row[2]].push({
ligne: row[4],
destination: row[5],
direction: row[6],
time: row[7],
theorique: row[8],
});
}
});
stream.on('end', () => res());
stream.on('error', err => rej(err));
res.send('Hello World!');
})
app.get('/line/:lineRef', async (req, res) =>
{
const lineData = await data.fetchLineData(req.params.lineRef);
res.json(lineData);
const networkData = await network.fetch(['1', '2', '3', '4']);
res.json(networkData);
});
app.listen(port, () => console.log(`Example app listening on port ${port}!`))