tracktracker/back/data/network.js

const geolib = require('geolib');

const {choosePlural, joinSentence} = require('../util');
const {queryOverpass, fetchTamTheoretical} = require('./endpoints');

const osmViewNode = 'https://www.openstreetmap.org/node';

/**
 * Create a link to remotely add tags into JOSM.
 *
 * @param objectId Identifier for the object to add the tags to.
 * @param tags Tags to add.
 * @return Link for remotely adding the tags.
 */
const josmAddTagToNode = (objectId, tags) =>
    'http://127.0.0.1:8111/load_object?' + [
        `objects=n${objectId}`,
        'new_layer=false',
        'addtags=' + tags.join('%7C'),
    ].join('&');

/**
 * Use theoretical passings data to guess which lines use which stops in which
 * direction.
 *
 * This is used for suggesting possible stop IDs for stops that don’t have
 * one in OSM.
 *
 * @return Map containing for each stop its abbreviated name, the lines that
 * use it and in which directions it is used.
 */
const fetchStopsRefAssociations = () => new Promise((res, rej) =>
{
    const stops = {};

    fetchTamTheoretical((err, row) =>
    {
        if (err)
        {
            rej(err);
            return;
        }

        if (!row)
        {
            res(stops);
            return;
        }

        let line = row.routeShortName;

        if (line === '4')
        {
            line += row.directionId === '0' ? 'A' : 'B';
        }

        if (!(row.stopId in stops))
        {
            stops[row.stopId] = {
                name: row.stopName,
                lines: new Set([line]),
                directions: new Set([row.tripHeadsign]),
            };
        }
        else
        {
            const stop = stops[row.stopId];

            if (stop.name !== row.stopName)
            {
                console.warn(`Stop ${row.stopId} has multiple names: \
“${row.stopName}” and “${stop.name}”. Only the first one will be considered.`);
            }

            stop.lines.add(line);
            stop.directions.add(row.tripHeadsign);
        }
    });
});

// Mapping for abbreviations used in stop names
const stopAbbreviations = {
    st: 'saint',
};

/**
 * Convert a stop name to a canonical representation suitable for
 * comparing two names.
 *
 * @param stopName Original stop name.
 * @return List of normalized tokens in the name.
 */
const canonicalizeStopName = stopName => stopName
    .toLowerCase()

    // Remove diacritics
    .normalize('NFD').replace(/[\u0300-\u036f]/g, '')

    // Only keep alpha-numeric characters
    .replace(/[^a-z0-9]/g, ' ')

    // Split in tokens longer than two characters
    .split(/\s+/g).filter(part => part.length >= 2)

    // Resolve well-known abbreviations
    .map(part => part in stopAbbreviations ? stopAbbreviations[part] : part);

/**
 * Compute a matching score between two stop names.
 *
 * @param fullName Stop name in full.
 * @param abbrName Abbreviated stop name.
 * @return Matching score (number of common tokens).
 */
const matchStopNames = (fullName, abbrName) =>
{
    const canonicalFullName = canonicalizeStopName(fullName);
    const canonicalAbbrName = canonicalizeStopName(abbrName);

    return canonicalFullName.filter(part =>
        canonicalAbbrName.findIndex(abbrPart =>
            part.startsWith(abbrPart)
        ) !== -1
    ).length;
};

/**
 * Fetch stops, segments and lines in the network.
 *
 * @param lineRefs List of lines to fetch.
 * @return Object with a set of stops, segments and lines.
 */
const fetch = async (lineRefs) =>
{
    // Retrieve routes, ways and stops from OpenStreetMap
    const rawData = await queryOverpass(`[out:json];

// Find the public transport line bearing the requested reference
relation[network="TaM"][type="route_master"][ref~"^(${lineRefs.join('|')})$"];

// Recursively fetch routes, ways and stops inside the line
(._; >>;);

out body qt;
`);

    // Retrieve stop associations from TaM
    const associations = await fetchStopsRefAssociations();

    // List of retrieved objects
    const elementsList = JSON.parse(rawData).elements;

    // List of retrieved lines
    const routeMasters = elementsList.filter(elt =>
        elt.tags && elt.tags.type === 'route_master'
    );

    // Retrieved objects indexed by ID
    const elements = elementsList.reduce((prev, elt) =>
    {
        prev[elt.id] = elt;
        return prev;
    }, {});

    // Result object containing all stops
    const stops = {};

    // Result object containing all segments between stops
    const segments = {};

    // Result object containing all lines
    const lines = {};

    for (let routeMaster of routeMasters)
    {
        const lineRef = routeMaster.tags.ref;
        const color = routeMaster.tags.colour || '#000000';

        // Extract all routes for the given line
        const rawRoutes = routeMaster.members.map(({ref}) => elements[ref]);

        // Add missing stops to the result object
        for (let route of rawRoutes)
        {
            for (let {ref, role} of route.members)
            {
                if (role === 'stop')
                {
                    const stop = elements[ref];

                    if (!('ref' in stop.tags))
                    {
                        console.warn(`Stop ${stop.id} is missing a “ref” tag
Name: ${stop.tags.name}
Part of line: ${route.tags.name}
URI: ${osmViewNode}/${stop.id}
`);

                        // Try to identify stops matching this stop in the
                        // TaM-provided data, using the stop name, line number
                        // and trip direction
                        const candidates = Object.entries(associations).filter(
                            ([, {lines}]) => lines.has(route.tags.ref)
                        ).map(([stopRef, {name, lines, directions}]) => ({
                            stopRef,
                            lines,

                            name,
                            nameScore: matchStopNames(stop.tags.name, name),

                            directions,
                            directionScore: Math.max(
                                ...Array.from(directions).map(direction =>
                                    matchStopNames(route.tags.to, direction)
                                )
                            ),
                        }))
                            // Only keep non-zero scores for both criteria
                            .filter(({nameScore, directionScore}) =>
                                nameScore && directionScore
                            )
                            // Sort by best name score then best direction
                            .sort(({
                                nameScore: nameScore1,
                                directionScore: directionScore1,
                            }, {
                                nameScore: nameScore2,
                                directionScore: directionScore2,
                            }) =>
                                (nameScore2 - nameScore1)
                                || (directionScore2 - directionScore1)
                            )
                            .slice(0, 4);

                        if (candidates.length === 0)
                        {
                            console.warn('No candidate found in TaM data.');
                        }
                        else
                        {
                            console.warn('Candidates:');

                            for (let candidate of candidates)
                            {
                                console.warn(`\
    — Stop ${candidate.stopRef} with name “${candidate.name}” used by \
${choosePlural(candidate.lines.length, 'line', '.s')} \
${joinSentence(Array.from(candidate.lines), ', ', ' and ')} going to \
${joinSentence(Array.from(candidate.directions), ', ', ' or ')}
      Apply in JOSM: ${josmAddTagToNode(stop.id, ['ref=' + candidate.stopRef])}
`);
                            }
                        }

                        console.warn('');
                    }

                    if (!(stop.tags.ref in stops))
                    {
                        stops[stop.tags.ref] = {
                            lat: stop.lat,
                            lon: stop.lon,
                            name: stop.tags.name,
                            lines: [lineRef],
                        };
                    }
                    else
                    {
                        stops[stop.tags.ref].lines.push(lineRef);
                    }
                }
            }
        }

        // Add missing segments between stops
        for (let route of rawRoutes)
        {
            const {from, to} = route.tags;

            const stops = route.members
                .filter(({role}) => role === 'stop')
                .map(({ref}) => elements[ref])
                .filter(stop => 'ref' in stop.tags)
                .map(stop => ({
                    id: stop.id,
                    lat: stop.lat,
                    lon: stop.lon,
                    ref: stop.tags.ref,
                    name: stop.tags.name,
                }));

            const ways = route.members
                .filter(({role}) => role === '')
                .map(({ref}) => ref);

            // Construct a graph with all connected nodes
            const nodeNeighbors = new Map();

            for (let id of ways)
            {
                const {type, nodes, tags} = elements[id];

                const isOneWay = (
                    tags.oneway === 'yes'
                    || tags.junction === 'roundabout'
                );

                const canGoBackward = (
                    !isOneWay
                    || parseInt(tags['lanes:psv:backward'], 10) > 0
                );

                if (type === 'way' && nodes.length >= 1)
                {
                    let previousNode = nodes[0];

                    if (!nodeNeighbors.has(previousNode))
                    {
                        nodeNeighbors.set(previousNode, new Set());
                    }

                    for (let node of nodes.slice(1))
                    {
                        if (!nodeNeighbors.has(node))
                        {
                            nodeNeighbors.set(node, new Set());
                        }

                        nodeNeighbors.get(previousNode).add(node);

                        if (canGoBackward)
                        {
                            nodeNeighbors.get(node).add(previousNode);
                        }

                        previousNode = node;
                    }
                }
            }

            // Find way from first stop through the end using DFS
            const numberOfStops = stops.length;
            let currentStopIndex = 0;

            while (currentStopIndex + 1 < numberOfStops)
            {
                const currentStop = stops[currentStopIndex];
                const nextStop = stops[currentStopIndex + 1];
                const segmentId = `${currentStop.ref}-${nextStop.ref}`;

                if (!(segmentId in segments))
                {
                    const visitedEdges = new Set();
                    const stack = [{
                        currentNode: currentStop.id,
                        segment: [currentStop.id],
                    }];

                    let found = false;

                    while (stack.length !== 0)
                    {
                        const {currentNode, segment} = stack.pop();

                        if (currentNode === nextStop.id)
                        {
                            // Arrived at next stop
                            segments[segmentId] = {
                                nodes: segment.map(id =>
                                {
                                    const {lat, lon} = elements[id];
                                    return {lat, lon};
                                }),
                                length: geolib.getPathLength(segment.map(id =>
                                {
                                    const {lat, lon} = elements[id];
                                    return {latitude: lat, longitude: lon};
                                })),
                                lines: [lineRef],
                            };

                            found = true;
                            break;
                        }

                        const neighbors = nodeNeighbors.get(currentNode) || [];

                        for (let nextNode of neighbors)
                        {
                            const edge = `${currentNode}-${nextNode}`;

                            if (!visitedEdges.has(edge))
                            {
                                visitedEdges.add(edge);
                                stack.push({
                                    currentNode: nextNode,
                                    segment: segment.concat([nextNode]),
                                });
                            }
                        }
                    }

                    if (!found)
                    {
                        throw new Error(`No way between stop \
“${currentStop.name}” (${currentStop.id}) and stop “${nextStop.name}” \
(${nextStop.id}) on line ${lineRef}’s route from “${from}” to “${to}”`);
                    }
                }
                else
                {
                    segments[segmentId].lines.push(lineRef);
                }

                ++currentStopIndex;
            }
        }

        // Construct line objects
        const routes = rawRoutes.map(route => ({
            from: route.tags.from,
            to: route.tags.to,

            // Retrieve each stop’s information (stop order in the relation is
            // assumed to reflect reality)
            stops: route.members
                .filter(({role}) => role === 'stop')
                .map(({ref}) => elements[ref])
                .filter(stop => 'ref' in stop.tags)
                .map(stop => stop.tags.ref),
        }));

        lines[lineRef] = {
            color,
            routes,
        };
    }

    return {stops, segments, lines};
};

exports.fetch = fetch;