tracktracker/back/data/network.js

/**
 * @file
 *
 * Extract static information about the TaM network from OpenStreetMap (OSM):
 * tram and bus lines, stops and routes.
 *
 * Functions in this file also report and offer to correct errors that may
 * occur in OSM data.
 *
 * Because of the static nature of this data, it is cached in a
 * version-controlled file `network.json` next to this file. To update it, use
 * the `script/update-network` script.
 */

const geolib = require('geolib');
const util = require('../util');
const osm = require('./sources/osm');
const tam = require('./sources/tam');

/**
 * Use theoretical passings data to guess which lines use which stops in which
 * direction.
 *
 * This is used for suggesting possible stop IDs for stops that don’t have
 * one in OSM.
 *
 * @return Map containing for each stop its abbreviated name, the lines that
 * use it and in which directions it is used.
 */
const fetchStopsRefAssociations = () => new Promise((res, rej) =>
{
    const stops = {};

    tam.fetchTheoretical((err, row) =>
    {
        if (err)
        {
            rej(err);
            return;
        }

        if (!row)
        {
            res(stops);
            return;
        }

        let line = row.routeShortName;

        if (line === '4')
        {
            line += row.directionId === '0' ? 'A' : 'B';
        }

        if (!(row.stopId in stops))
        {
            stops[row.stopId] = {
                name: row.stopName,
                lines: new Set([line]),
                directions: new Set([row.tripHeadsign]),
            };
        }
        else
        {
            const stop = stops[row.stopId];

            if (stop.name !== row.stopName)
            {
                console.warn(`Stop ${row.stopId} has multiple names: \
“${row.stopName}” and “${stop.name}”. Only the first one will be considered.`);
            }

            stop.lines.add(line);
            stop.directions.add(row.tripHeadsign);
        }
    });
});

// Mapping for abbreviations used in stop names
const stopAbbreviations = {
    st: 'saint',
};

/**
 * Convert a stop name to a canonical representation suitable for
 * comparing two names.
 *
 * @param stopName Original stop name.
 * @return List of normalized tokens in the name.
 */
const canonicalizeStopName = stopName => stopName
    .toLowerCase()

    // Remove diacritics
    .normalize('NFD').replace(/[\u0300-\u036f]/g, '')

    // Only keep alpha-numeric characters
    .replace(/[^a-z0-9]/g, ' ')

    // Split in tokens longer than two characters
    .split(/\s+/g).filter(part => part.length >= 2)

    // Resolve well-known abbreviations
    .map(part => part in stopAbbreviations ? stopAbbreviations[part] : part);

/**
 * Compute a matching score between two stop names.
 *
 * @param fullName Stop name in full.
 * @param abbrName Abbreviated stop name.
 * @return Matching score (number of common tokens).
 */
const matchStopNames = (fullName, abbrName) =>
{
    const canonicalFullName = canonicalizeStopName(fullName);
    const canonicalAbbrName = canonicalizeStopName(abbrName);

    return canonicalFullName.filter(part =>
        canonicalAbbrName.findIndex(abbrPart =>
            part.startsWith(abbrPart)
        ) !== -1
    ).length;
};

/**
 * Fetch stops and lines of the network.
 *
 * @param lineRefs List of lines to fetch.
 * @return Object with a set of stops, segments and lines.
 */
const fetch = async (lineRefs) =>
{
    // Retrieve routes, ways and stops from OpenStreetMap
    const rawData = await osm.runQuery(`[out:json];

// Find the public transport line bearing the requested reference
relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"];

// Recursively fetch routes, ways and stops inside the line
(._; >>;);

out body qt;
`);

    // Retrieve stop associations from TaM
    const associations = await fetchStopsRefAssociations();

    // List of retrieved objects
    const elementsList = rawData.elements;

    // List of retrieved lines
    const routeMasters = elementsList.filter(osm.isTransportLine);

    // Retrieved objects indexed by ID
    const elements = elementsList.reduce((prev, elt) =>
    {
        prev[elt.id] = elt;
        return prev;
    }, {});

    // All stops in the network
    const stops = {};

    // All transport lines of the network
    const lines = {};

    // All segments leading from one stop to another
    const segments = {};

    for (let routeMaster of routeMasters)
    {
        const lineRef = routeMaster.tags.ref;
        const color = routeMaster.tags.colour || '#000000';

        // Extract all routes for the given line
        const rawRoutes = routeMaster.members.map(({ref}) => elements[ref]);

        // Add missing stops to the result object
        for (let route of rawRoutes)
        {
            for (let {ref, role} of route.members)
            {
                if (role === 'stop')
                {
                    const stop = elements[ref];

                    if (!('ref' in stop.tags))
                    {
                        console.warn(`Stop ${stop.id} is missing a “ref” tag
Name: ${stop.tags.name}
Part of line: ${route.tags.name}
URI: ${osm.viewNode(stop.id)}
`);

                        // Try to identify stops matching this stop in the
                        // TaM-provided data, using the stop name, line number
                        // and trip direction
                        const candidates = Object.entries(associations).filter(
                            ([, {lines}]) => lines.has(route.tags.ref)
                        ).map(([stopRef, {name, lines, directions}]) => ({
                            stopRef,
                            lines,

                            name,
                            nameScore: matchStopNames(stop.tags.name, name),

                            directions,
                            directionScore: Math.max(
                                ...Array.from(directions).map(direction =>
                                    matchStopNames(route.tags.to, direction)
                                )
                            ),
                        }))
                            // Only keep non-zero scores for both criteria
                            .filter(({nameScore, directionScore}) =>
                                nameScore && directionScore
                            )
                            // Sort by best name score then best direction
                            .sort(({
                                nameScore: nameScore1,
                                directionScore: directionScore1,
                            }, {
                                nameScore: nameScore2,
                                directionScore: directionScore2,
                            }) =>
                                (nameScore2 - nameScore1)
                                || (directionScore2 - directionScore1)
                            )
                            .slice(0, 4);

                        if (candidates.length === 0)
                        {
                            console.warn('No candidate found in TaM data.');
                        }
                        else
                        {
                            console.warn('Candidates:');

                            for (let candidate of candidates)
                            {
                                console.warn(`\
    — Stop ${candidate.stopRef} with name “${candidate.name}” used by \
${util.choosePlural(candidate.lines.length, 'line', '.s')} \
${util.joinSentence(Array.from(candidate.lines), ', ', ' and ')} going to \
${util.joinSentence(Array.from(candidate.directions), ', ', ' or ')}
      Apply in JOSM: ${osm.addTagsToNode(stop.id, ['ref=' + candidate.stopRef])}
`);
                            }
                        }

                        console.warn('');
                    }

                    if (!(stop.tags.ref in stops))
                    {
                        stops[stop.tags.ref] = {
                            lat: stop.lat,
                            lon: stop.lon,
                            name: stop.tags.name,
                            lines: new Set([lineRef]),
                        };
                    }
                    else
                    {
                        stops[stop.tags.ref].lines.add(lineRef);
                    }
                }
            }
        }

        // Reconstruct the line’s route from stop to stop
        const routes = [];

        for (let route of rawRoutes)
        {
            const {from, to, name} = route.tags;

            // Check that the route consists of a block of stops and platforms
            // followed by a block of routes as dictated by PTv2
            const relationPivot = route.members.findIndex(
                ({role}) => role === ''
            );

            if (!route.members.slice(0, relationPivot).every(
                ({role}) => role === 'stop' || role === 'platform'
            ))
            {
                throw new Error(`Members with invalid roles in between stops
of ${name}`);
            }

            if (!route.members.slice(relationPivot).every(
                ({role}) => role === ''
            ))
            {
                throw new Error(`Members with invalid roles inside the path
of ${name}`);
            }

            // List of stops in the route, expected to be in the timetable
            // order as per PTv2 and to be traversed in order by the sequence
            // of ways extracted below
            const stops = route.members.slice(0, relationPivot)
                .filter(({role}) => role === 'stop')
                .map(({ref}) => ref);

            // List of ways making up the route’s path through its stops
            // with each way connected to the next through a single point
            const ways = route.members.slice(relationPivot)
                .map(({ref}) => ref);

            // Merge all used ways in a single path
            let path = [];
            let currentNode = stops[0];

            for (let wayIndex = 0; wayIndex < ways.length; wayIndex += 1)
            {
                const way = elements[ways[wayIndex]];
                const {nodes: wayNodes, tags: wayTags} = way;
                const wayNodesSet = new Set(wayNodes);

                const curNodeIndex = wayNodes.indexOf(currentNode);

                // If not the last way, find a connection point to the next way
                // (there should be exactly one)
                let nextNode = null;
                let nextNodeIndex = null;

                if (wayIndex + 1 < ways.length)
                {
                    const nextNodeCandidates = elements[ways[wayIndex + 1]]
                        .nodes.filter(node => wayNodesSet.has(node));

                    if (nextNodeCandidates.length !== 1)
                    {
                        throw new Error(`There should be exactly one point
connecting way n°${wayIndex} and way n°${wayIndex + 1} in ${name},
but there are ${nextNodeCandidates.length}`);
                    }

                    nextNode = nextNodeCandidates[0];
                    nextNodeIndex = wayNodes.indexOf(nextNode);
                }
                else
                {
                    nextNodeIndex = wayNodes.length;
                }

                if (curNodeIndex < nextNodeIndex)
                {
                    // Use the way in its normal direction
                    path = path.concat(
                        wayNodes.slice(curNodeIndex, nextNodeIndex)
                    );
                }
                else
                {
                    // Use the way in the reverse direction
                    if (osm.isOneWay(way))
                    {
                        throw new Error(`Way n°${wayIndex} in
${name} is one-way and cannot be used in reverse.`);
                    }

                    path = path.concat(
                        wayNodes.slice(nextNodeIndex + 1, curNodeIndex + 1)
                            .reverse()
                    );
                }

                currentNode = nextNode;
            }

            // Split the path into segments between stops
            for (let stopIndex = 0; stopIndex + 1 < stops.length; ++stopIndex)
            {
                const begin = elements[stops[stopIndex]].tags.ref;
                const end = elements[stops[stopIndex + 1]].tags.ref;

                const id = `${begin}-${end}`;
                const nodes = path.slice(
                    path.indexOf(stops[stopIndex]),
                    path.indexOf(stops[stopIndex + 1]) + 1,
                );

                if (id in segments)
                {
                    if (!util.arraysEqual(nodes, segments[id].nodes))
                    {
                        throw new Error(`Segment ${id} is defined as a
different sequence of nodes in two or more lines.`);
                    }

                    segments[id].lines.add(lineRef);
                }
                else
                {
                    const points = nodes.map(id => ({
                        lat: elements[id].lat,
                        lon: elements[id].lon
                    }));

                    if (points.length)
                    {
                        // Augment each point with the distance to the start
                        points[0].distance = 0;

                        for (let i = 1; i < points.length; ++i)
                        {
                            const len = geolib.getPreciseDistance(
                                ...[points[i - 1], points[i]]
                                    .map(({lat, lon}) => ({
                                        latitude: lat,
                                        longitude: lon
                                    })),
                            );

                            points[i].distance = points[i - 1].distance + len;
                        }
                    }

                    segments[id] = {
                        // Keep track of the original sequence of nodes to
                        // compare with duplicates
                        nodes,
                        points,
                        lines: new Set([lineRef]),
                    };
                }
            }

            routes.push({
                from, to, name,
                stops: stops.map(id => elements[id].tags.ref),
            });
        }

        lines[lineRef] = {
            color,
            routes,
        };
    }

    // Remove OSM nodes from segments that were only used for checking validity
    for (let segment of Object.values(segments))
    {
        delete segment.nodes;
    }

    return {stops, lines, segments};
};

exports.fetch = fetch;