From a4e1ee199bafd6fae3fc9bf08cdd2c7e76bb34a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matt=C3=A9o=20Delabre?= Date: Tue, 14 Jan 2020 14:08:08 +0100 Subject: [PATCH] Retrieve whole network and start of realtime --- back/data.js | 198 -------------------------------- back/data/network.js | 254 ++++++++++++++++++++++++++++++++++++++++++ back/data/realtime.js | 172 ++++++++++++++++++++++++++++ front/index.js | 63 +++++------ package-lock.json | 5 + package.json | 1 + server.js | 55 +-------- 7 files changed, 464 insertions(+), 284 deletions(-) delete mode 100644 back/data.js create mode 100644 back/data/network.js create mode 100644 back/data/realtime.js diff --git a/back/data.js b/back/data.js deleted file mode 100644 index f7275a5..0000000 --- a/back/data.js +++ /dev/null @@ -1,198 +0,0 @@ -const requestp = require('request-promise-native'); - -const {makeCached} = require('./util'); - -const OVERPASS_ENDPOINT = 'https://lz4.overpass-api.de/api/interpreter'; - -const fetchLineData = makeCached(async (lineRef) => -{ - // Retrieve routes, ways and stops from OpenStreetMap - const rawData = await requestp.post(OVERPASS_ENDPOINT, {form: `\ -data=[out:json]; - -// Find the public transport line bearing the requested reference -relation[network="TaM"][type="route_master"][ref="${lineRef}"]; - -// Recursively fetch routes, ways and stops inside the line -(._; >>;); - -out body qt; - `}); - - const elementsList = JSON.parse(rawData).elements; - - // Extract all routes for the given line - const rawRoutes = elementsList.filter(elt => - elt.type === 'relation' - && elt.tags.type === 'route' - && elt.tags.ref === lineRef - ); - - // If no route is found, assume the line does not exist - if (rawRoutes.length === 0) - { - return null; - } - - // Index retrieved objects by their ID - const elements = elementsList.reduce((prev, elt) => - { - prev[elt.id] = elt; - return prev; - }, {}); - - const color = rawRoutes[0].tags.colour || '#000000'; - - // Extract stops for each route of the line - const routes = rawRoutes.map(route => ({ - from: route.tags.from, - to: route.tags.to, - - // Retrieve each stop’s information (stop order in the relation is - // assumed to reflect reality) - stops: route.members - .filter(({role}) => role === 'stop') - .map(({ref}) => { - const elt = elements[ref]; - return { - id: ref, - lat: elt.lat, - lon: elt.lon, - name: elt.tags.name, - }; - }), - - // List of ways making up the route (raw) - rawWays: route.members - .filter(({role}) => role === '') - .map(({ref}) => ref) - })); - - // Process ways in each route to sort and merge them - for (let route of routes) - { - const {from, to, stops, rawWays} = route; - - // Construct a graph with all connected nodes - const nodeNeighbors = new Map(); - - for (let id of rawWays) - { - const {type, nodes, tags} = elements[id]; - - const isOneWay = ( - tags.oneway === 'yes' - || tags.junction === 'roundabout' - ); - - const canGoBackward = ( - !isOneWay - || parseInt(tags['lanes:psv:backward'], 10) > 0 - ); - - if (type === 'way' && nodes.length >= 1) - { - let previousNode = nodes[0]; - - if (!nodeNeighbors.has(previousNode)) - { - nodeNeighbors.set(previousNode, new Set()); - } - - for (let node of nodes.slice(1)) - { - if (!nodeNeighbors.has(node)) - { - nodeNeighbors.set(node, new Set()); - } - - nodeNeighbors.get(previousNode).add(node); - - if (canGoBackward) - { - nodeNeighbors.get(node).add(previousNode); - } - - previousNode = node; - } - } - } - - // Find way from first stop through the end using DFS - const numberOfStops = stops.length; - const ways = []; - - let currentStopIndex = 0; - - while (currentStopIndex + 1 < numberOfStops) - { - const currentStop = stops[currentStopIndex]; - const nextStop = stops[currentStopIndex + 1]; - - const visitedEdges = new Set(); - const stack = [{ - currentNode: currentStop.id, - way: [currentStop.id], - }]; - - let found = false; - - while (stack.length !== 0) - { - const {currentNode, way} = stack.pop(); - - if (currentNode === nextStop.id) - { - // Arrived at next stop - ways.push(way); - found = true; - break; - } - - const neighbors = nodeNeighbors.get(currentNode) || []; - - for (let nextNode of neighbors) - { - const edge = `${currentNode}-${nextNode}`; - - if (!visitedEdges.has(edge)) - { - visitedEdges.add(edge); - stack.push({ - currentNode: nextNode, - way: way.concat([nextNode]), - }); - } - } - } - - if (!found) - { - throw new Error(`No way between stop “${currentStop.name}” \ -(${currentStop.id}) and stop “${nextStop.name}” (${nextStop.id}) on line \ -${lineRef}’s route from “${from}” to “${to}”`); - } - - ++currentStopIndex; - } - - // Only keep geo coordinates for each way node - route.ways = ways.map(way => - way.map(id => - { - const node = elements[id]; - return {lat: node.lat, lon: node.lon}; - }) - ); - - delete route.rawWays; - } - - return { - ref: lineRef, - color, - routes - }; -}); - -exports.fetchLineData = fetchLineData; diff --git a/back/data/network.js b/back/data/network.js new file mode 100644 index 0000000..6554435 --- /dev/null +++ b/back/data/network.js @@ -0,0 +1,254 @@ +const requestp = require('request-promise-native'); +const geolib = require('geolib'); + +const {makeCached} = require('../util'); + +const OVERPASS_ENDPOINT = 'https://lz4.overpass-api.de/api/interpreter'; + +const fetch = makeCached(async (lineRefs) => +{ + // Retrieve routes, ways and stops from OpenStreetMap + const rawData = await requestp.post(OVERPASS_ENDPOINT, {form: `\ +data=[out:json]; + +// Find the public transport line bearing the requested reference +relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"]; + +// Recursively fetch routes, ways and stops inside the line +(._; >>;); + +out body qt; + `}); + + // List of retrieved objects + const elementsList = JSON.parse(rawData).elements; + + // List of retrieved lines + const routeMasters = elementsList.filter(elt => + elt.tags && elt.tags.type === 'route_master' + ); + + // Retrieved objects indexed by ID + const elements = elementsList.reduce((prev, elt) => + { + prev[elt.id] = elt; + return prev; + }, {}); + + // Result object containing all stops + const stops = {}; + + // Result object containing all segments between stops + const segments = {}; + + // Result object containing all lines + const lines = {}; + + for (let routeMaster of routeMasters) + { + const lineRef = routeMaster.tags.ref; + const color = routeMaster.tags.colour || '#000000'; + + // Extract all routes for the given line + const rawRoutes = routeMaster.members.map(({ref}) => elements[ref]); + + // Add missing stops to the result object + for (let route of rawRoutes) + { + for (let {ref, role} of route.members) + { + if (role === 'stop') + { + const stop = elements[ref]; + + if (!('ref' in stop.tags)) + { + console.warn(`Stop ${stop.id} is missing a “ref” tag`); + continue; + } + + if (!(stop.tags.ref in stops)) + { + stops[stop.tags.ref] = { + lat: stop.lat, + lon: stop.lon, + name: stop.tags.name, + lines: [lineRef], + }; + } + else + { + stops[stop.tags.ref].lines.push(lineRef); + } + } + } + } + + // Add missing segments between stops + for (let route of rawRoutes) + { + const {from, to} = route.tags; + + const stops = route.members + .filter(({role}) => role === 'stop') + .map(({ref}) => elements[ref]) + .filter(stop => 'ref' in stop.tags) + .map(stop => ({ + id: stop.id, + lat: stop.lat, + lon: stop.lon, + ref: stop.tags.ref, + name: stop.tags.name, + })); + + const ways = route.members + .filter(({role}) => role === '') + .map(({ref}) => ref); + + // Construct a graph with all connected nodes + const nodeNeighbors = new Map(); + + for (let id of ways) + { + const {type, nodes, tags} = elements[id]; + + const isOneWay = ( + tags.oneway === 'yes' + || tags.junction === 'roundabout' + ); + + const canGoBackward = ( + !isOneWay + || parseInt(tags['lanes:psv:backward'], 10) > 0 + ); + + if (type === 'way' && nodes.length >= 1) + { + let previousNode = nodes[0]; + + if (!nodeNeighbors.has(previousNode)) + { + nodeNeighbors.set(previousNode, new Set()); + } + + for (let node of nodes.slice(1)) + { + if (!nodeNeighbors.has(node)) + { + nodeNeighbors.set(node, new Set()); + } + + nodeNeighbors.get(previousNode).add(node); + + if (canGoBackward) + { + nodeNeighbors.get(node).add(previousNode); + } + + previousNode = node; + } + } + } + + // Find way from first stop through the end using DFS + const numberOfStops = stops.length; + let currentStopIndex = 0; + + while (currentStopIndex + 1 < numberOfStops) + { + const currentStop = stops[currentStopIndex]; + const nextStop = stops[currentStopIndex + 1]; + const segmentId = `${currentStop.ref}-${nextStop.ref}`; + + if (!(segmentId in segments)) + { + const visitedEdges = new Set(); + const stack = [{ + currentNode: currentStop.id, + segment: [currentStop.id], + }]; + + let found = false; + + while (stack.length !== 0) + { + const {currentNode, segment} = stack.pop(); + + if (currentNode === nextStop.id) + { + // Arrived at next stop + segments[segmentId] = { + nodes: segment.map(id => + { + const {lat, lon} = elements[id]; + return {lat, lon}; + }), + length: geolib.getPathLength(segment.map(id => + { + const {lat, lon} = elements[id]; + return {latitude: lat, longitude: lon}; + })), + lines: [lineRef], + }; + + found = true; + break; + } + + const neighbors = nodeNeighbors.get(currentNode) || []; + + for (let nextNode of neighbors) + { + const edge = `${currentNode}-${nextNode}`; + + if (!visitedEdges.has(edge)) + { + visitedEdges.add(edge); + stack.push({ + currentNode: nextNode, + segment: segment.concat([nextNode]), + }); + } + } + } + + if (!found) + { + throw new Error(`No way between stop \ +“${currentStop.name}” (${currentStop.id}) and stop “${nextStop.name}” \ +(${nextStop.id}) on line ${lineRef}’s route from “${from}” to “${to}”`); + } + } + else + { + segments[segmentId].lines.push(lineRef); + } + + ++currentStopIndex; + } + } + + // Construct line objects + const routes = rawRoutes.map(route => ({ + from: route.tags.from, + to: route.tags.to, + + // Retrieve each stop’s information (stop order in the relation is + // assumed to reflect reality) + stops: route.members + .filter(({role}) => role === 'stop') + .map(({ref}) => elements[ref]) + .filter(stop => 'ref' in stop.tags) + .map(stop => stop.tags.ref) + })); + + lines[lineRef] = { + color, + routes + }; + } + + return {stops, segments, lines}; +}); + +exports.fetch = fetch; diff --git a/back/data/realtime.js b/back/data/realtime.js new file mode 100644 index 0000000..900aa73 --- /dev/null +++ b/back/data/realtime.js @@ -0,0 +1,172 @@ +const request = require('request'); +const csv = require('csv-parse'); + +const network = require('./network'); + +const TAM_REALTIME = 'http://data.montpellier3m.fr/node/10732/download'; + +const sortByFirstKey = (a, b) => a[0] - b[0]; + +const fetchRealtime = () => new Promise((res, rej) => +{ + const parser = csv({ + delimiter: ';', + }); + + const stream = request(TAM_REALTIME).pipe(parser); + const courses = {}; + + stream.on('readable', () => + { + let row; + + while (row = stream.read()) + { + if (row.length === 0 || row[0] === 'course') + { + // Ignore les lignes invalides et l’en-tête + continue; + } + + const course = row[0]; + const stopRef = row[2]; + const lineRef = row[4]; + const eta = row[9]; + const destinationRef = row[10]; + + if (!(course in courses)) + { + courses[course] = { + lineRef, + destinationRef, + stops: [], + }; + } + + courses[course].stops.push([parseInt(eta, 10), stopRef]); + courses[course].stops.sort(sortByFirstKey); + } + }); + + stream.on('end', () => res(courses)); + stream.on('error', err => rej(err)); +}); + +const updateVehicles = async (lines, vehicles) => +{ + const courses = await fetchRealtime(); + const currentTime = Math.floor(Date.now() / 1000); + + for (let [courseRef, course] of Object.entries(courses)) + { + if (course.lineRef in lines) + { + if (!(courseRef in vehicles)) + { + // New vehicle: identify which route it pertains to + const line = lines[course.lineRef]; + let routeIndex = null; + + for (let [index, route] of Object.entries(line.routes)) + { + const destRef = route.stops[route.stops.length - 1].ref; + + if (destRef === course.destinationRef) + { + routeIndex = index; + } + } + + if (routeIndex !== null) + { + const route = line.routes[routeIndex]; + + // Convert ETAs to absolute times + const nextStops = course.stops.map(([eta, ref]) => [ + eta + currentTime, + ref + ]); + + // Convert stop refs to indices + const stopIndices = course.stops.map(([eta, ref]) => [ + eta, + ]); + + // Find the preceding stop from which the vehicle is coming + const arrivingStop = stopIndices[0][1]; + const arrivingStopIndex = + route.stops.findIndex(stop => stop.ref === arrivingStop); + + + + const [eta, nextStop] = stopIndices[0]; + + if (nextStop === 0) + { + // Vehicle at starting point + vehicles[courseRef] = { + lineRef: course.lineRef, + stopRef + + stopIndex: 0, + nextStops: stopIndices, + + distance: 0, + speed: 0, + }; + } + else + { + // Vehicle in transit between two stops + vehicles[courseRef] = { + lineRef: course.lineRef, + routeIndex, + + stopIndex: nextStop - 1, + nextStops: stopIndices, + + distance: 0, + speed: route.distances[nextStop - 1] / eta, + }; + } + } + } + else + { + // Existing vehicle: update information + const vehicle = vehicles[courseRef]; + + const line = lines[vehicle.lineRef]; + const route = line.routes[vehicle.routeIndex]; + + // Convert stop refs to indices + const stopIndices = course.stops.map(([eta, ref]) => [ + eta, + route.stops.findIndex(stop => stop.ref === ref), + ]); + + console.log(stopIndices); + console.log(vehicle); + console.log(course); + console.log('---'); + } + } + } +}; + +const sleep = time => new Promise(res => setTimeout(res, time)); + +const updateLoop = async (lines, vehicles = {}) => +{ + await updateVehicles(lines, vehicles); + await sleep(30000); + return updateLoop(lines, vehicles); +}; + +(async () => +{ + const lines = {'1': await network.fetchLineData('1')}; + updateLoop(lines); + + // console.log(require('util').inspect(vehicles, true, 10)); +})(); diff --git a/front/index.js b/front/index.js index 2e04839..0a949ff 100644 --- a/front/index.js +++ b/front/index.js @@ -32,53 +32,44 @@ licenses/by-sa/2.0/">CC-BY-SA`, const makeBorderColor = mainColor => { const hsl = color(mainColor).hsl(); - - if (hsl.color[2] < 40) - { - hsl.color[2] += 30; - } - else - { - hsl.color[2] -= 20; - } - + hsl.color = Math.max(0, hsl.color[2] -= 20); return hsl.hex(); }; const SERVER = window.origin; -fetch(SERVER + '/line/1').then(res => res.json()).then(line => +fetch(SERVER + '/network').then(res => res.json()).then(network => { - const color = line.color; - const borderColor = makeBorderColor(color); - - line.routes.forEach(route => + Object.values(network.segments).forEach(segment => { - route.ways.forEach(way => - { - const wayPoints = way.map(({lat, lon}) => [lat, lon]); + const color = network.lines[segment.lines[0]].color; + const borderColor = makeBorderColor(color); + const nodes = segment.nodes.map(({lat, lon}) => [lat, lon]); - L.polyline(wayPoints, {weight: 8, color: borderColor}).addTo(map); - L.polyline(wayPoints, {weight: 6, color}).addTo(map); - }); + L.polyline(nodes, {weight: 8, color: borderColor}).addTo(map); + const line = L.polyline(nodes, {weight: 6, color}).addTo(map); + + line.bindPopup(`${segment.length} m`); }); - line.routes.forEach(route => + Object.values(network.stops).forEach(stop => { - route.stops.forEach(stop => - { - const stopMarker = L.circleMarker( - [stop.lat, stop.lon], - { - fillColor: color, - radius: 6, - fillOpacity: 1, - color: borderColor, - weight: 2 - } - ).addTo(map); + const color = network.lines[stop.lines[0]].color; + const borderColor = makeBorderColor(color); - stopMarker.bindPopup(stop.name); - }); + const stopMarker = L.circleMarker( + [stop.lat, stop.lon], + { + fillColor: color, + radius: 6, + fillOpacity: 1, + color: borderColor, + weight: 2 + } + ).addTo(map); + + stopMarker.bindPopup(stop.name); }); + + console.log(network); }); diff --git a/package-lock.json b/package-lock.json index bc09b08..e4668e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3616,6 +3616,11 @@ "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz", "integrity": "sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg==" }, + "geolib": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/geolib/-/geolib-3.2.1.tgz", + "integrity": "sha512-O9nD8iSD4VimupKak8bKySLkkWI5VWetxIXsU7jmJRXxBFRR9LxSXGfTomtcHJLSRiznx+YHXHTOIVq4qgQmPw==" + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", diff --git a/package.json b/package.json index 7e6acbb..e32008c 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "color": "^3.1.2", "csv-parse": "^4.8.3", "express": "^4.17.1", + "geolib": "^3.2.1", "leaflet": "^1.6.0", "parcel-bundler": "^1.12.4", "request": "^2.88.0", diff --git a/server.js b/server.js index 0444072..5fdcf95 100644 --- a/server.js +++ b/server.js @@ -1,65 +1,20 @@ const express = require('express'); -const request = require('request'); -const csv = require('csv-parse'); -const data = require('./back/data'); +const network = require('./back/data/network'); const app = express(); const port = 3000; app.use(express.static('dist')); -const TAM_REALTIME = 'http://data.montpellier3m.fr/node/10732/download'; - app.get('/realtime', (req, res) => { - const parser = csv({ - delimiter: ';', - }); +}); - passages = {}; - passagesLast = Date.now(); - - const stream = request(TAM_ENDPOINT) - .pipe(parser); - - stream.on('readable', () => - { - let row; - - while (row = stream.read()) - { - if (row.length === 0 || row[0] === 'course') - { - // Ignore les lignes invalides - continue; - } - - if (passages[row[2]] === undefined) - { - passages[row[2]] = []; - } - - passages[row[2]].push({ - ligne: row[4], - destination: row[5], - direction: row[6], - time: row[7], - theorique: row[8], - }); - } - }); - - stream.on('end', () => res()); - stream.on('error', err => rej(err)); - - res.send('Hello World!'); -}) - -app.get('/line/:lineRef', async (req, res) => +app.get('/network', async (req, res) => { - const lineData = await data.fetchLineData(req.params.lineRef); - res.json(lineData); + const networkData = await network.fetch(['1', '2', '3', '4']); + res.json(networkData); }); app.listen(port, () => console.log(`Example app listening on port ${port}!`))