#!/usr/bin/env bash

set -Eeuo pipefail

# ==============================================================================
# NextLimit
# Build automatique des datasets SQLite v2 préindexés pour l'Europe
#
# Flux :
#
# Geofabrik .osm.pbf
#       ↓
# osm-dataset/build_dataset.py
#       ↓
# SQLite NextLimit v2
#       ↓
# graphe + RTree + indexes
#       ↓
# validations serveur
#       ↓
# gzip reproductible (niveau 9, mtime=0)
#       ↓
# SHA-256 du .sqlite.gz
#       ↓
# packages/{ID}-{VERSION}.sqlite.gz
# packages/{ID}-{VERSION}.sqlite.gz.sha256
# packages/{ID}.json
#       ↓
# catalog.json v2 publié après chaque package, puis revalidé en fin de série
#
# Dépendance hôte :
#
#   Docker uniquement
#
# Arborescence attendue :
#
# /var/www/dl/osm/
# ├── build-europe-preindexed.sh
# └── osm-dataset/
#     ├── build_dataset.py
#     ├── dataset_schema.py
#     ├── requirements.txt
#     ├── test_build_dataset.py
#     └── test_dataset_schema.py
#
# Usage :
#
#   ./build-europe-preindexed.sh
#
# Tests ciblés :
#
#   COUNTRIES="AD" ./build-europe-preindexed.sh
#   COUNTRIES="LU,BE" ./build-europe-preindexed.sh
#   COUNTRIES="FR" ./build-europe-preindexed.sh
#
# Reconstruire une révision déjà publiée :
#
#   DATASET_REVISION=3 COUNTRIES="FR" ./build-europe-preindexed.sh
#
# Conserver les PBF :
#
#   KEEP_SOURCE=1 ./build-europe-preindexed.sh
#
# Conserver les SQLite :
#
#   KEEP_SQLITE=1 ./build-europe-preindexed.sh
#
# ==============================================================================


# ==============================================================================
# Configuration hôte
# ==============================================================================

SCRIPT_DIR="$(
    cd "$(dirname "${BASH_SOURCE[0]}")"
    pwd
)"

OSM_DATASET_DIR="${OSM_DATASET_DIR:-${SCRIPT_DIR}/osm-dataset}"

BUILD_ROOT="${BUILD_ROOT:-${SCRIPT_DIR}/build/osm-europe}"

PUBLIC_ROOT="${PUBLIC_ROOT:-${SCRIPT_DIR}/public-osm/data}"

IMAGE_NAME="${IMAGE_NAME:-nextlimit-osm-builder:latest}"

INDEX_URL="${INDEX_URL:-https://download.geofabrik.de/index-v1-nogeom.json}"

COUNTRIES="${COUNTRIES:-}"

FORCE="${FORCE:-0}"

KEEP_SOURCE="${KEEP_SOURCE:-0}"

KEEP_SQLITE="${KEEP_SQLITE:-0}"

MAX_PACKAGES="${MAX_PACKAGES:-0}"

DATASET_REVISION="${DATASET_REVISION:-2}"

RUN_PIPELINE_TESTS="${RUN_PIPELINE_TESTS:-1}"

LIST_ONLY="${LIST_ONLY:-0}"


# ==============================================================================
# Helpers hôte
# ==============================================================================

log()
{
    printf \
        '\n\033[1;34m[NextLimit]\033[0m %s\n' \
        "$*"
}


success()
{
    printf \
        '\n\033[1;32m[NextLimit]\033[0m %s\n' \
        "$*"
}


warning()
{
    printf \
        '\n\033[1;33m[NextLimit]\033[0m %s\n' \
        "$*"
}


die()
{
    printf \
        '\n\033[1;31m[ERREUR]\033[0m %s\n' \
        "$*" >&2

    exit 1
}


# ==============================================================================
# Vérifications hôte
# ==============================================================================

command -v docker >/dev/null 2>&1 \
    || die "Docker n'est pas installé."

docker info >/dev/null 2>&1 \
    || die "Docker n'est pas démarré ou inaccessible."

[[ -f "${OSM_DATASET_DIR}/build_dataset.py" ]] \
    || die "build_dataset.py introuvable : ${OSM_DATASET_DIR}"

[[ -f "${OSM_DATASET_DIR}/dataset_schema.py" ]] \
    || die "dataset_schema.py introuvable : ${OSM_DATASET_DIR}"

[[ -f "${OSM_DATASET_DIR}/requirements.txt" ]] \
    || die "requirements.txt introuvable : ${OSM_DATASET_DIR}"

[[ -f "${SCRIPT_DIR}/Dockerfile" ]] \
    || die "Dockerfile introuvable : ${SCRIPT_DIR}"

[[ "${DATASET_REVISION}" =~ ^[1-9][0-9]*$ ]] \
    || die "DATASET_REVISION doit être un entier positif."

mkdir -p \
    "${BUILD_ROOT}" \
    "${PUBLIC_ROOT}"


# ==============================================================================
# Image Docker
# ==============================================================================

log "Construction de l'image Docker de compilation..."


docker build \
    --tag "${IMAGE_NAME}" \
    --file "${SCRIPT_DIR}/Dockerfile" \
    "${SCRIPT_DIR}"


# ==============================================================================
# Pipeline Docker
# ==============================================================================

log "Lancement du pipeline Europe..."


docker run \
    --rm \
    --interactive \
    --user "$(id -u):$(id -g)" \
    --env "INDEX_URL=${INDEX_URL}" \
    --env "COUNTRIES=${COUNTRIES}" \
    --env "FORCE=${FORCE}" \
    --env "KEEP_SOURCE=${KEEP_SOURCE}" \
    --env "KEEP_SQLITE=${KEEP_SQLITE}" \
    --env "MAX_PACKAGES=${MAX_PACKAGES}" \
    --env "DATASET_REVISION=${DATASET_REVISION}" \
    --env "RUN_PIPELINE_TESTS=${RUN_PIPELINE_TESTS}" \
    --env "LIST_ONLY=${LIST_ONLY}" \
    --volume "${OSM_DATASET_DIR}:/opt/nextlimit/osm-dataset:ro" \
    --volume "${BUILD_ROOT}:/build-osm" \
    --volume "${PUBLIC_ROOT}:/public-osm" \
    "${IMAGE_NAME}" \
    bash -s <<'CONTAINER_SCRIPT'

set -Eeuo pipefail


# ==============================================================================
# Répertoires conteneur
# ==============================================================================

TOOLS="/opt/nextlimit/osm-dataset"

BUILD="/build-osm"

SOURCES="${BUILD}/sources"

SQLITES="${BUILD}/sqlite"

REPORTS="${BUILD}/reports"

TMP="${BUILD}/tmp"

PUBLIC="/public-osm"

PACKAGES="${PUBLIC}/packages"

INDEX_JSON="${BUILD}/geofabrik-index.json"

COUNTRY_LIST="${BUILD}/europe-packages.tsv"

SUMMARY="${BUILD}/build-summary.csv"

FAILED="${BUILD}/failed.tsv"


mkdir -p \
    "${SOURCES}" \
    "${SQLITES}" \
    "${REPORTS}" \
    "${TMP}" \
    "${PACKAGES}"


: > "${FAILED}"


# ==============================================================================
# Helpers conteneur
# ==============================================================================

log()
{
    printf \
        '\n\033[1;34m[Builder]\033[0m %s\n' \
        "$*"
}


success()
{
    printf \
        '\n\033[1;32m[Builder]\033[0m %s\n' \
        "$*"
}


warning()
{
    printf \
        '\n\033[1;33m[Builder]\033[0m %s\n' \
        "$*"
}


human_size()
{
    numfmt \
        --to=iec-i \
        --suffix=B \
        "$1"
}


# ==============================================================================
# Autotests du pipeline livré
# ==============================================================================

if [[ "${RUN_PIPELINE_TESTS:-1}" == "1" ]]
then
    log "Tests unitaires du pipeline..."

    (
        cd "${TOOLS}"
        PYTHONDONTWRITEBYTECODE=1 \
            python3 -m unittest \
                test_dataset_schema.py \
                test_build_dataset.py
    )
fi


# ==============================================================================
# Versions
# ==============================================================================

log "Versions utilisées"

echo

python3 --version

osmium --version

sqlite3 --version

gzip --version | head -1

echo


# ==============================================================================
# Vérification RTree
# ==============================================================================

log "Vérification SQLite RTree..."


RTREE_TEST="$(
    sqlite3 ':memory:' \
        "CREATE VIRTUAL TABLE test_rtree USING rtree(id,minX,maxX,minY,maxY); SELECT rtreecheck('test_rtree');"
)"


[[ "${RTREE_TEST}" == "ok" ]] \
    || {
        echo "RTree SQLite indisponible."
        exit 1
    }


# ==============================================================================
# Index Geofabrik
# ==============================================================================

log "Téléchargement de l'index Geofabrik..."


curl \
    --fail \
    --location \
    --retry 10 \
    --retry-delay 5 \
    --output "${INDEX_JSON}" \
    "${INDEX_URL}"


# ==============================================================================
# Détection des pays européens
# ==============================================================================

log "Détection des packages Europe..."


python3 - \
    "${INDEX_JSON}" \
    "${COUNTRY_LIST}" \
    "${COUNTRIES}" <<'PY'

import json
import sys


index_file = sys.argv[1]
output_file = sys.argv[2]
requested_raw = sys.argv[3].strip()


requested = {
    item.strip().upper()
    for item in requested_raw.split(",")
    if item.strip()
}


with open(
    index_file,
    "r",
    encoding="utf-8",
) as handle:

    payload = json.load(handle)


features = payload.get(
    "features",
    [],
)


by_id = {}


for feature in features:

    properties = feature.get(
        "properties",
        {},
    )

    feature_id = properties.get(
        "id"
    )

    if feature_id:

        by_id[feature_id] = properties


priority_regions = {
    "france": (0, "FR", "France"),
    "guyane": (1, "GF", "Guyane"),
    "guadeloupe": (2, "GP", "Guadeloupe"),
    "martinique": (3, "MQ", "Martinique"),
    "reunion": (4, "RE", "La Réunion"),
    "mayotte": (5, "YT", "Mayotte"),
    "new-caledonia": (6, "NC", "Nouvelle-Calédonie"),
    "polynesie-francaise": (7, "PF", "Polynésie française"),
    "wallis-et-futuna": (8, "WF", "Wallis-et-Futuna"),
    "ile-de-clipperton": (9, "CP", "Île de Clipperton"),
}


def belongs_to_europe(properties):

    feature_id = properties.get(
        "id",
        "",
    )

    if feature_id == "europe":

        return True

    parent = properties.get(
        "parent"
    )

    visited = set()

    while parent:

        if parent in visited:

            break

        visited.add(parent)

        if parent == "europe":

            return True

        parent_properties = by_id.get(
            parent
        )

        if parent_properties is None:

            break

        parent = parent_properties.get(
            "parent"
        )

    return False


rows = []


for properties in by_id.values():

    region_id = properties.get(
        "id"
    )

    priority_region = priority_regions.get(
        region_id
    )

    if not belongs_to_europe(
        properties
    ) and priority_region is None:

        continue


    if priority_region is not None:

        priority, priority_iso, priority_name = priority_region

        iso_codes = [priority_iso]

        name = priority_name

    else:

        priority = 100

        iso_codes = (
            properties.get(
                "iso3166-1:alpha2"
            )
            or []
        )

        name = (
            properties.get(
                "name"
            )
            or region_id
        )


    if not iso_codes:

        continue


    iso_codes = [
        str(code).upper()
        for code in iso_codes
    ]


    if requested:

        if not requested.intersection(
            iso_codes
        ):

            continue


    urls = properties.get(
        "urls"
    ) or {}


    pbf_url = urls.get(
        "pbf"
    )


    if not pbf_url:

        continue


    package_id = "-".join(
        sorted(
            iso_codes
        )
    )


    rows.append(
        (
            priority,
            package_id,
            name,
            ",".join(
                iso_codes
            ),
            region_id,
            pbf_url,
        )
    )


rows.sort(
    key=lambda value: (value[0], value[1])
)


with open(
    output_file,
    "w",
    encoding="utf-8",
) as handle:

    for row in rows:

        handle.write(
            "\t".join(
                str(value)
                for value in row[1:]
            )
            + "\n"
        )


print(
    f"{len(rows)} package(s) détecté(s)"
)


for row in rows:

    print(
        f"{row[1]:10s} {row[2]}"
    )

PY


if [[ "${LIST_ONLY:-0}" == "1" ]]
then
    success "Liste des packages générée, aucun dataset construit."
    exit 0
fi


# ==============================================================================
# Rapport global
# ==============================================================================

echo \
"id,name,version,source_bytes,sqlite_bytes,gzip_bytes,segments,speed_control_nodes,speed_control_relations,resolved_speed_control_relations,speed_control_approach_segments,build_seconds,publish_seconds,status" \
> "${SUMMARY}"


TOTAL=0
BUILT=0
SKIPPED=0
ERRORS=0


# ==============================================================================
# Publication atomique du catalogue après chaque package
# ==============================================================================

publish_catalog_incremental()
{
    python3 - \
        "${PACKAGES}" \
        "${PUBLIC}/catalog.json" <<'PY'

import datetime
import json
import os
import pathlib
import sys


packages_dir = pathlib.Path(sys.argv[1])
catalog_file = pathlib.Path(sys.argv[2])
catalog_staging = catalog_file.with_suffix(".json.staging")


def fail(message):
    raise SystemExit(message)


def safe_file_name(value):
    return (
        value
        and pathlib.Path(value).name == value
        and "/" not in value
        and "\\" not in value
        and ".." not in value
        and "://" not in value
    )


packages = []


for manifest_path in sorted(packages_dir.glob("*.json")):
    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    required = (
        "id",
        "name",
        "version",
        "format",
        "compression",
        "schemaVersion",
        "file",
        "checksumFile",
        "size",
        "installedSize",
        "generatedAt",
    )
    missing = [key for key in required if manifest.get(key) is None]
    if missing:
        fail(f"{manifest_path.name}: champs manquants {missing}")

    package_id = str(manifest["id"])
    package_name = str(manifest["file"])
    checksum_name = str(manifest["checksumFile"])
    if not safe_file_name(package_name) or not safe_file_name(checksum_name):
        fail(f"{package_id}: nom de fichier non sûr")
    if manifest["format"] != "nextlimit_sqlite" or manifest["compression"] != "gzip":
        fail(f"{package_id}: format de package non supporté")
    if int(manifest["schemaVersion"]) != 2:
        fail(f"{package_id}: schemaVersion non supportée")

    package_path = packages_dir / package_name
    checksum_path = packages_dir / checksum_name
    if not package_path.is_file() or not checksum_path.is_file():
        fail(f"{package_id}: package ou checksum absent")
    if package_path.stat().st_size != int(manifest["size"]):
        fail(f"{package_id}: taille package incohérente")
    checksum_parts = checksum_path.read_text(encoding="utf-8").split()
    expected_checksum = checksum_parts[0].lower() if checksum_parts else ""
    if len(expected_checksum) != 64 or any(character not in "0123456789abcdef" for character in expected_checksum):
        fail(f"{package_id}: fichier checksum invalide")

    package_id_parts = package_id.split("-")
    fallback_iso_codes = (
        package_id_parts
        if package_id_parts and all(len(code) == 2 and code.isalpha() for code in package_id_parts)
        else []
    )
    iso_codes = manifest.get("iso3166_1") or fallback_iso_codes
    if not iso_codes:
        fail(f"{package_id}: iso3166_1 absent")

    entry = {
        "id": package_id,
        "name": str(manifest["name"]),
        "iso3166_1": iso_codes,
        "geofabrikRegionId": manifest.get("geofabrikRegionId"),
        "version": str(manifest["version"]),
        "format": "nextlimit_sqlite",
        "compression": "gzip",
        "schemaVersion": 2,
        "file": package_name,
        "checksumFile": checksum_name,
        "size": int(manifest["size"]),
        "installedSize": int(manifest["installedSize"]),
        "generatedAt": manifest["generatedAt"],
    }
    optional_integer_fields = (
        "datasetRevision",
        "segmentCount",
        "speedControlNodeCount",
        "speedControlRelationCount",
        "resolvedSpeedControlRelationCount",
        "speedControlApproachSegmentCount",
    )
    for key in optional_integer_fields:
        if manifest.get(key) is not None:
            entry[key] = int(manifest[key])
    if manifest.get("sourceTimestamp"):
        entry["sourceTimestamp"] = manifest["sourceTimestamp"]
    packages.append(entry)


packages.sort(key=lambda package: package["id"])
catalog = {
    "version": 2,
    "region": "europe",
    "generatedAt": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
    "count": len(packages),
    "packages": packages,
}
serialized = json.dumps(catalog, ensure_ascii=False, indent=2) + "\n"
if "download.geofabrik.de" in serialized:
    fail("URL Geofabrik interdite dans le catalogue public")
catalog_staging.write_text(serialized, encoding="utf-8")
os.replace(catalog_staging, catalog_file)
print(f"Catalogue publié : {len(packages)} package(s)")

PY
}


# ==============================================================================
# Traitement pays par pays
# ==============================================================================

while IFS=$'\t' read -r \
    PACKAGE_ID \
    PACKAGE_NAME \
    ISO_CODES \
    GEOFABRIK_REGION \
    PBF_URL

do

    [[ -z "${PACKAGE_ID}" ]] && continue


    TOTAL=$((TOTAL + 1))


    if [[ "${MAX_PACKAGES:-0}" -gt 0 ]] \
        && [[ "${TOTAL}" -gt "${MAX_PACKAGES}" ]]
    then

        break

    fi


    echo
    echo "================================================================"
    echo "${PACKAGE_ID} - ${PACKAGE_NAME}"
    echo "================================================================"


    # ==========================================================================
    # Contexte réglementaire
    # ==========================================================================

    IFS=',' read -ra ISO_ARRAY <<< "${ISO_CODES}"


    if [[ "${#ISO_ARRAY[@]}" -eq 1 ]]
    then

        COUNTRY_CODE="${ISO_ARRAY[0]}"

    else

        COUNTRY_CODE="XX"

    fi


    SOURCE_FILE="${SOURCES}/${PACKAGE_ID}-latest.osm.pbf"


    # ==========================================================================
    # Téléchargement
    # ==========================================================================

    log "${PACKAGE_ID}: téléchargement Geofabrik"


    curl \
        --fail \
        --location \
        --retry 10 \
        --retry-delay 5 \
        --retry-all-errors \
        --continue-at - \
        --output "${SOURCE_FILE}" \
        "${PBF_URL}"


    osmium fileinfo \
        "${SOURCE_FILE}" \
        >/dev/null


    SOURCE_BYTES="$(
        stat -c%s "${SOURCE_FILE}"
    )"


    # ==========================================================================
    # Timestamp OSM
    # ==========================================================================

    SOURCE_TIMESTAMP="$(
        osmium fileinfo \
            -g header.option.osmosis_replication_timestamp \
            "${SOURCE_FILE}" \
            2>/dev/null \
            || true
    )"


    if [[ -z "${SOURCE_TIMESTAMP}" ]]
    then

        SOURCE_TIMESTAMP="$(
            osmium fileinfo \
                -g header.option.timestamp \
                "${SOURCE_FILE}" \
                2>/dev/null \
                || true
        )"

    fi


    if [[ -z "${SOURCE_TIMESTAMP}" ]]
    then

        warning "${PACKAGE_ID}: timestamp PBF absent."

        SOURCE_TIMESTAMP="$(
            date \
                -u \
                '+%Y-%m-%dT00:00:00Z'
        )"

    fi


    # ==========================================================================
    # Version immuable YYYY.MM.DD.REVISION
    # ==========================================================================

    SOURCE_DATE_VERSION="$(
        python3 \
            - "${SOURCE_TIMESTAMP}" <<'PY'

import datetime
import sys


value = sys.argv[1]


try:

    value = value.replace(
        "Z",
        "+00:00",
    )

    parsed = datetime.datetime.fromisoformat(
        value
    )

    print(
        parsed.strftime(
            "%Y.%m.%d"
        )
    )

except Exception:

    print(
        datetime.datetime.now(
            datetime.timezone.utc
        ).strftime(
            "%Y.%m.%d"
        )
    )

PY
    )"

    VERSION="${SOURCE_DATE_VERSION}.${DATASET_REVISION}"


    DATASET_ID="${PACKAGE_ID}-${VERSION}"

    SQLITE_FILE="${SQLITES}/${PACKAGE_ID}.sqlite"

    REPORT_FILE="${REPORTS}/${PACKAGE_ID}-${VERSION}.report.json"

    PACKAGE_FILE="${PACKAGE_ID}-${VERSION}.sqlite.gz"

    CHECKSUM_FILE="${PACKAGE_FILE}.sha256"

    MANIFEST_FILE="${PACKAGE_ID}.json"

    EXPECTED_PACKAGE="${PACKAGES}/${PACKAGE_FILE}"

    EXPECTED_SHA="${PACKAGES}/${CHECKSUM_FILE}"

    EXPECTED_MANIFEST="${PACKAGES}/${MANIFEST_FILE}"


    if [[ "${FORCE:-0}" == "1" ]] \
        && { [[ -e "${EXPECTED_PACKAGE}" ]] \
            || [[ -e "${EXPECTED_SHA}" ]] \
            || [[ -e "${EXPECTED_MANIFEST}" ]]; }
    then
        echo \
            "${PACKAGE_ID} ${VERSION} existe déjà. " \
            "Incrémentez DATASET_REVISION pour préserver les URLs immuables."
        exit 1
    fi


    # ==========================================================================
    # Skip package existant
    # ==========================================================================

    if [[ "${FORCE:-0}" != "1" ]] \
        && [[ -s "${EXPECTED_PACKAGE}" ]] \
        && [[ -s "${EXPECTED_SHA}" ]] \
        && [[ -s "${EXPECTED_MANIFEST}" ]]

    then

        if (
            cd "${PACKAGES}"

            sha256sum \
                -c \
                "${CHECKSUM_FILE}" \
                >/dev/null
        )
        then

            success \
                "${PACKAGE_ID} ${VERSION} déjà publié."

            publish_catalog_incremental

            SKIPPED=$((SKIPPED + 1))


            if [[ "${KEEP_SOURCE:-0}" != "1" ]]
            then

                rm -f \
                    "${SOURCE_FILE}"

            fi


            continue

        fi

    fi


    # ==========================================================================
    # Build SQLite v2
    # ==========================================================================

    rm -f \
        "${SQLITE_FILE}"


    BUILD_STARTED="$(
        date +%s
    )"


    log \
        "${PACKAGE_ID}: génération SQLite v2"


    python3 \
        "${TOOLS}/build_dataset.py" \
        "${SOURCE_FILE}" \
        "${SQLITE_FILE}" \
        --dataset-id "${DATASET_ID}" \
        --country "${COUNTRY_CODE}" \
        --region "${PACKAGE_ID}" \
        --source-date "${SOURCE_TIMESTAMP}" \
        --report "${REPORT_FILE}" \
        --force


    BUILD_ENDED="$(
        date +%s
    )"


    BUILD_SECONDS=$((BUILD_ENDED - BUILD_STARTED))


    SQLITE_BYTES="$(
        stat -c%s "${SQLITE_FILE}"
    )"


    # ==========================================================================
    # Validation SQLite
    # ==========================================================================

    log \
        "${PACKAGE_ID}: validation SQLite"


    QUICK_CHECK="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "PRAGMA quick_check(1);"
    )"


    [[ "${QUICK_CHECK}" == "ok" ]] \
        || {
            echo \
                "PRAGMA quick_check a échoué."

            exit 1
        }


    SCHEMA_VERSION="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT schema_version FROM dataset_metadata LIMIT 1;"
    )"


    [[ "${SCHEMA_VERSION}" == "2" ]] \
        || {
            echo \
                "schema_version=${SCHEMA_VERSION}, attendu=2"

            exit 1
        }


    REGION_CODE="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT region_code FROM dataset_metadata LIMIT 1;"
    )"


    [[ "${REGION_CODE}" == "${PACKAGE_ID}" ]] \
        || {
            echo \
                "region_code incorrect : ${REGION_CODE}"

            exit 1
        }


    META_SEGMENTS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT segment_count FROM dataset_metadata LIMIT 1;"
    )"


    ROAD_SEGMENTS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT COUNT(*) FROM road_segments;"
    )"


    RTREE_SEGMENTS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT COUNT(*) FROM road_segment_rtree;"
    )"


    [[ "${META_SEGMENTS}" -gt 0 ]] \
        || {
            echo \
                "segment_count doit être > 0."

            exit 1
        }


    [[ "${META_SEGMENTS}" == "${ROAD_SEGMENTS}" ]] \
        || {
            echo \
                "segment_count != road_segments (${META_SEGMENTS}/${ROAD_SEGMENTS})."

            exit 1
        }


    [[ "${META_SEGMENTS}" == "${RTREE_SEGMENTS}" ]] \
        || {
            echo \
                "segment_count != road_segment_rtree (${META_SEGMENTS}/${RTREE_SEGMENTS})."

            exit 1
        }


    RTREE_CHECK="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT rtreecheck('road_segment_rtree');"
    )"


    [[ "${RTREE_CHECK}" == "ok" ]] \
        || {
            echo \
                "rtreecheck a échoué : ${RTREE_CHECK}"

            exit 1
        }


    for REQUIRED_INDEX in \
        road_segments_from_node_idx \
        road_segments_to_node_idx \
        road_segments_way_idx

    do

        INDEX_COUNT="$(
            sqlite3 \
                "${SQLITE_FILE}" \
                "
                SELECT COUNT(*)
                FROM sqlite_master
                WHERE
                    type = 'index'
                    AND name = '${REQUIRED_INDEX}';
                "
        )"


        [[ "${INDEX_COUNT}" == "1" ]] \
            || {
                echo \
                    "Index manquant : ${REQUIRED_INDEX}"

                exit 1
            }

    done


    MISSING_RTREE="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "
            SELECT COUNT(*)
            FROM road_segments s
            LEFT JOIN road_segment_rtree r
                ON r.row_id = s.row_id
            WHERE r.row_id IS NULL;
            "
    )"


    [[ "${MISSING_RTREE}" == "0" ]] \
        || {
            echo \
                "${MISSING_RTREE} segment(s) sans RTree."

            exit 1
        }


    INVALID_SPEEDS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "
            SELECT COUNT(*)
            FROM road_segments
            WHERE
                (
                    speed_limit_kmh IS NOT NULL
                    AND (
                        speed_limit_kmh < 5
                        OR speed_limit_kmh > 250
                    )
                )
                OR speed_source < 0
                OR speed_source > 4
                OR speed_confidence < 0
                OR speed_confidence > 100;
            "
    )"


    [[ "${INVALID_SPEEDS}" == "0" ]] \
        || {
            echo \
                "${INVALID_SPEEDS} limitation(s) incohérente(s)."

            exit 1
        }


    INVALID_ROAD_FLAGS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT COUNT(*) FROM road_segments WHERE (road_flags & -8) != 0;"
    )"


    [[ "${INVALID_ROAD_FLAGS}" == "0" ]] \
        || {
            echo \
                "${INVALID_ROAD_FLAGS} segment(s) avec des flags inconnus."

            exit 1
        }


    SPEED_CONTROL_NODES="$(
        jq -r \
            '.filtering.speedControlNodes // 0' \
            "${REPORT_FILE}"
    )"


    SPEED_CONTROL_RELATIONS="$(
        jq -r \
            '.filtering.speedControlRelations // 0' \
            "${REPORT_FILE}"
    )"


    RESOLVED_SPEED_CONTROL_RELATIONS="$(
        jq -r \
            '.filtering.resolvedSpeedControlRelations // 0' \
            "${REPORT_FILE}"
    )"


    SPEED_CONTROL_APPROACH_SEGMENTS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT COUNT(*) FROM road_segments WHERE (road_flags & 4) != 0;"
    )"


    OSM_ATTRIBUTION="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT osm_attribution FROM dataset_metadata LIMIT 1;"
    )"


    OSM_LICENSE="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT osm_license FROM dataset_metadata LIMIT 1;"
    )"


    [[ -n "${OSM_ATTRIBUTION}" ]] \
        || {
            echo \
                "Attribution OSM absente."

            exit 1
        }


    [[ "${OSM_LICENSE}" == "ODbL-1.0" ]] \
        || {
            echo \
                "Licence OSM incorrecte : ${OSM_LICENSE}"

            exit 1
        }


    # ==========================================================================
    # Publication NextLimit v2
    # ==========================================================================

    PUBLISH_STAGE="$(
        mktemp \
            -d \
            "${TMP}/${PACKAGE_ID}.publish.XXXXXX"
    )"


    # Nettoyage même en cas d'erreur dans la phase suivante.
    cleanup_publish_stage()
    {
        if [[ -n "${PUBLISH_STAGE:-}" ]] \
            && [[ -d "${PUBLISH_STAGE}" ]]
        then

            rm -rf \
                "${PUBLISH_STAGE}"

        fi
    }


    STAGED_PACKAGE="${PUBLISH_STAGE}/${PACKAGE_FILE}"

    STAGED_SHA="${PUBLISH_STAGE}/${CHECKSUM_FILE}"

    STAGED_MANIFEST="${PUBLISH_STAGE}/${MANIFEST_FILE}"


    PUBLISH_STARTED="$(
        date +%s
    )"


    # ==========================================================================
    # Gzip reproductible
    # ==========================================================================

    log \
        "${PACKAGE_ID}: compression gzip reproductible"


    python3 - \
        "${SQLITE_FILE}" \
        "${STAGED_PACKAGE}" <<'PY'

import gzip
import shutil
import sys


source = sys.argv[1]
destination = sys.argv[2]


with open(
    source,
    "rb",
) as src:

    with open(
        destination,
        "wb",
    ) as raw_dst:

        with gzip.GzipFile(
            filename="",
            mode="wb",
            fileobj=raw_dst,
            compresslevel=9,
            mtime=0,
        ) as dst:

            shutil.copyfileobj(
                src,
                dst,
                length=1024 * 1024,
            )

PY


    [[ -s "${STAGED_PACKAGE}" ]] \
        || {
            cleanup_publish_stage

            echo \
                "Le gzip n'a pas été généré."

            exit 1
        }


    PUBLISH_ENDED="$(
        date +%s
    )"


    PUBLISH_SECONDS=$((PUBLISH_ENDED - PUBLISH_STARTED))


    GZIP_BYTES="$(
        stat -c%s "${STAGED_PACKAGE}"
    )"


    # ==========================================================================
    # Validation gzip
    # ==========================================================================

    log \
        "${PACKAGE_ID}: validation gzip"


    gzip \
        --test \
        "${STAGED_PACKAGE}"


    ORIGINAL_SQLITE_SHA="$(
        sha256sum \
            "${SQLITE_FILE}" \
        | awk \
            '{print $1}'
    )"


    DECOMPRESSED_SQLITE_SHA="$(
        gzip \
            -dc \
            "${STAGED_PACKAGE}" \
        | sha256sum \
        | awk \
            '{print $1}'
    )"


    [[ "${ORIGINAL_SQLITE_SHA}" == "${DECOMPRESSED_SQLITE_SHA}" ]] \
        || {
            cleanup_publish_stage

            echo \
                "Le gzip ne restitue pas exactement le SQLite source."

            exit 1
        }


    # ==========================================================================
    # SHA-256 du gzip
    # ==========================================================================

    (
        cd "${PUBLISH_STAGE}"

        sha256sum \
            "${PACKAGE_FILE}" \
            > "${CHECKSUM_FILE}"
    )


    [[ -s "${STAGED_SHA}" ]] \
        || {
            cleanup_publish_stage

            echo \
                "Checksum non généré."

            exit 1
        }


    (
        cd "${PUBLISH_STAGE}"

        sha256sum \
            -c \
            "${CHECKSUM_FILE}"
    )


    # ==========================================================================
    # Metadata SQLite
    # ==========================================================================

    GENERATED_AT="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT generated_at FROM dataset_metadata LIMIT 1;"
    )"


    DB_SOURCE_TIMESTAMP="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT source_timestamp FROM dataset_metadata LIMIT 1;"
    )"


    SEGMENT_COUNT="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT segment_count FROM dataset_metadata LIMIT 1;"
    )"


    # ==========================================================================
    # Manifest
    # ==========================================================================

    log \
        "${PACKAGE_ID}: génération du manifest"


    python3 - \
        "${STAGED_MANIFEST}" \
        "${PACKAGE_ID}" \
        "${VERSION}" \
        "${DATASET_REVISION}" \
        "${PACKAGE_NAME}" \
        "${ISO_CODES}" \
        "${GEOFABRIK_REGION}" \
        "${PACKAGE_FILE}" \
        "${CHECKSUM_FILE}" \
        "${GZIP_BYTES}" \
        "${SQLITE_BYTES}" \
        "${GENERATED_AT}" \
        "${DB_SOURCE_TIMESTAMP}" \
        "${SEGMENT_COUNT}" \
        "${SPEED_CONTROL_NODES}" \
        "${SPEED_CONTROL_RELATIONS}" \
        "${RESOLVED_SPEED_CONTROL_RELATIONS}" \
        "${SPEED_CONTROL_APPROACH_SEGMENTS}" <<'PY'

import json
import sys


(
    output,
    package_id,
    version,
    dataset_revision,
    name,
    iso_codes,
    geofabrik_region_id,
    package_file,
    checksum_file,
    gzip_size,
    installed_size,
    generated_at,
    source_timestamp,
    segment_count,
    speed_control_nodes,
    speed_control_relations,
    resolved_speed_control_relations,
    speed_control_approach_segments,
) = sys.argv[1:]


manifest = {

    "id": package_id,

    "name": name,

    "version": version,

    "datasetRevision": int(dataset_revision),

    "iso3166_1": iso_codes.split(","),

    "geofabrikRegionId": geofabrik_region_id,

    "format": "nextlimit_sqlite",

    "compression": "gzip",

    "schemaVersion": 2,

    "file": package_file,

    "checksumFile": checksum_file,

    "size": int(
        gzip_size
    ),

    "installedSize": int(
        installed_size
    ),

    "generatedAt": generated_at,

    "sourceTimestamp": source_timestamp,

    "segmentCount": int(
        segment_count
    ),

    "speedControlNodeCount": int(
        speed_control_nodes
    ),

    "speedControlRelationCount": int(
        speed_control_relations
    ),

    "resolvedSpeedControlRelationCount": int(
        resolved_speed_control_relations
    ),

    "speedControlApproachSegmentCount": int(
        speed_control_approach_segments
    ),
}


with open(
    output,
    "w",
    encoding="utf-8",
) as handle:

    json.dump(
        manifest,
        handle,
        ensure_ascii=False,
        indent=2,
    )

    handle.write(
        "\n"
    )

PY


    # ==========================================================================
    # Validation manifest
    # ==========================================================================

    python3 - \
        "${STAGED_MANIFEST}" \
        "${PACKAGE_ID}" \
        "${VERSION}" \
        "${DATASET_REVISION}" \
        "${GZIP_BYTES}" \
        "${SQLITE_BYTES}" \
        "${SEGMENT_COUNT}" \
        "${SPEED_CONTROL_APPROACH_SEGMENTS}" <<'PY'

import json
import sys


(
    manifest_file,
    package_id,
    version,
    dataset_revision,
    gzip_size,
    sqlite_size,
    segment_count,
    speed_control_approach_segments,
) = sys.argv[1:]


with open(
    manifest_file,
    "r",
    encoding="utf-8",
) as handle:

    data = json.load(
        handle
    )


expected = {

    "id": package_id,

    "version": version,

    "format": "nextlimit_sqlite",

    "compression": "gzip",

    "schemaVersion": 2,

    "datasetRevision": int(dataset_revision),
}


for key, expected_value in expected.items():

    actual = data.get(
        key
    )

    if actual != expected_value:

        raise SystemExit(
            f"{key}: "
            f"{actual!r} != "
            f"{expected_value!r}"
        )


if int(
    data["size"]
) != int(
    gzip_size
):

    raise SystemExit(
        "size incorrect"
    )


if int(
    data["installedSize"]
) != int(
    sqlite_size
):

    raise SystemExit(
        "installedSize incorrect"
    )


if int(
    data["segmentCount"]
) != int(
    segment_count
):

    raise SystemExit(
        "segmentCount incorrect"
    )


if int(
    data["speedControlApproachSegmentCount"]
) != int(
    speed_control_approach_segments
):

    raise SystemExit(
        "speedControlApproachSegmentCount incorrect"
    )


if not data["file"].endswith(
    ".sqlite.gz"
):

    raise SystemExit(
        "Extension package invalide"
    )

PY


    # ==========================================================================
    # Publication atomique
    #
    # IMPORTANT :
    #
    # Le PUBLISH_STAGE n'est utilisé QU'ICI.
    #
    # Après ces mv, aucun ancien bloc ne doit essayer d'y relire quoi que ce soit.
    # ==========================================================================

    log \
        "${PACKAGE_ID}: publication atomique"


    mv \
        "${STAGED_PACKAGE}" \
        "${PACKAGES}/${PACKAGE_FILE}"


    mv \
        "${STAGED_SHA}" \
        "${PACKAGES}/${CHECKSUM_FILE}"


    mv \
        "${STAGED_MANIFEST}" \
        "${PACKAGES}/${MANIFEST_FILE}"


    publish_catalog_incremental


    cleanup_publish_stage


    success \
        "${PACKAGE_ID}: package publié."


    # ==========================================================================
    # Rapport
    # ==========================================================================

    printf \
        '"%s","%s","%s",%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"OK"\n' \
        "${PACKAGE_ID}" \
        "${PACKAGE_NAME//\"/\"\"}" \
        "${VERSION}" \
        "${SOURCE_BYTES}" \
        "${SQLITE_BYTES}" \
        "${GZIP_BYTES}" \
        "${META_SEGMENTS}" \
        "${SPEED_CONTROL_NODES}" \
        "${SPEED_CONTROL_RELATIONS}" \
        "${RESOLVED_SPEED_CONTROL_RELATIONS}" \
        "${SPEED_CONTROL_APPROACH_SEGMENTS}" \
        "${BUILD_SECONDS}" \
        "${PUBLISH_SECONDS}" \
        >> "${SUMMARY}"


    echo

    echo \
        "Source PBF : $(human_size "${SOURCE_BYTES}")"

    echo \
        "SQLite     : $(human_size "${SQLITE_BYTES}")"

    echo \
        "Gzip       : $(human_size "${GZIP_BYTES}")"

    echo \
        "Segments   : ${META_SEGMENTS}"

    echo \
        "Contrôles  : ${SPEED_CONTROL_APPROACH_SEGMENTS} approche(s), ${SPEED_CONTROL_NODES} nœud(s), ${SPEED_CONTROL_RELATIONS} relation(s)"

    echo \
        "Build      : ${BUILD_SECONDS} s"

    echo \
        "Compression: ${PUBLISH_SECONDS} s"


    # ==========================================================================
    # Nettoyage
    # ==========================================================================

    if [[ "${KEEP_SOURCE:-0}" != "1" ]]
    then

        rm -f \
            "${SOURCE_FILE}"

    fi


    if [[ "${KEEP_SQLITE:-0}" != "1" ]]
    then

        rm -f \
            "${SQLITE_FILE}"

    fi


    BUILT=$((BUILT + 1))


    success \
        "${PACKAGE_ID} ${VERSION} terminé."


done < "${COUNTRY_LIST}"


# ==============================================================================
# Catalogue public v2
# ==============================================================================

log \
    "Génération catalog.json v2..."


CATALOG_TMP="${PUBLIC}/catalog.json.tmp"

CATALOG_FINAL="${PUBLIC}/catalog.json"


python3 - \
    "${PACKAGES}" \
    "${COUNTRY_LIST}" \
    "${CATALOG_TMP}" <<'PY'

import datetime
import json
import pathlib
import sys


packages_dir = pathlib.Path(
    sys.argv[1]
)

country_list = pathlib.Path(
    sys.argv[2]
)

output = pathlib.Path(
    sys.argv[3]
)


country_metadata = {}


def fallback_iso_codes(package_id):

    values = package_id.split("-")

    if values and all(
        len(value) == 2
        and value.isalpha()
        for value in values
    ):

        return values

    return []


with country_list.open(
    "r",
    encoding="utf-8",
) as handle:

    for line in handle:

        line = line.rstrip(
            "\n"
        )

        if not line:

            continue

        (
            package_id,
            name,
            iso_codes,
            region_id,
            _source_url,
        ) = line.split(
            "\t"
        )

        country_metadata[
            package_id
        ] = {

            "name": name,

            "iso3166_1": iso_codes.split(
                ","
            ),

            "geofabrikRegionId": region_id,
        }


packages = []


for manifest_path in sorted(
    packages_dir.glob(
        "*.json"
    )
):

    with manifest_path.open(
        "r",
        encoding="utf-8",
    ) as handle:

        manifest = json.load(
            handle
        )


    package_id = manifest[
        "id"
    ]


    metadata = country_metadata.get(
        package_id,
        {},
    )


    package = {

        "id": package_id,

        "name": metadata.get(
            "name",
            manifest.get(
                "name",
                package_id,
            ),
        ),

        "iso3166_1": metadata.get(
            "iso3166_1",
            manifest.get(
                "iso3166_1",
            ) or fallback_iso_codes(
                package_id
            ),
        ),

        "geofabrikRegionId": metadata.get(
            "geofabrikRegionId",
            manifest.get(
                "geofabrikRegionId"
            ),
        ),

        "version": manifest[
            "version"
        ],

        "datasetRevision": int(
            manifest.get(
                "datasetRevision",
                1,
            )
        ),

        "format": manifest[
            "format"
        ],

        "compression": manifest[
            "compression"
        ],

        "schemaVersion": int(
            manifest[
                "schemaVersion"
            ]
        ),

        "file": manifest[
            "file"
        ],

        "checksumFile": manifest[
            "checksumFile"
        ],

        "size": int(
            manifest[
                "size"
            ]
        ),

        "installedSize": int(
            manifest[
                "installedSize"
            ]
        ),

        "generatedAt": manifest[
            "generatedAt"
        ],
    }


    if manifest.get(
        "sourceTimestamp"
    ):

        package[
            "sourceTimestamp"
        ] = manifest[
            "sourceTimestamp"
        ]


    if manifest.get(
        "segmentCount"
    ) is not None:

        package[
            "segmentCount"
        ] = int(
            manifest[
                "segmentCount"
            ]
        )


    for metric_name in (
        "speedControlNodeCount",
        "speedControlRelationCount",
        "resolvedSpeedControlRelationCount",
        "speedControlApproachSegmentCount",
    ):

        if manifest.get(metric_name) is not None:

            package[metric_name] = int(
                manifest[metric_name]
            )


    packages.append(
        package
    )


catalog = {

    "version": 2,

    "region": "europe",

    "generatedAt": (
        datetime.datetime.now(
            datetime.timezone.utc
        )
        .replace(
            microsecond=0
        )
        .isoformat()
        .replace(
            "+00:00",
            "Z",
        )
    ),

    "count": len(
        packages
    ),

    "packages": packages,
}


serialized = json.dumps(
    catalog,
    ensure_ascii=False,
    indent=2,
)


# Aucun lien source Geofabrik dans le catalogue mobile.
if "download.geofabrik.de" in serialized:

    raise SystemExit(
        "URL Geofabrik détectée dans catalog.json"
    )


with output.open(
    "w",
    encoding="utf-8",
) as handle:

    handle.write(
        serialized
    )

    handle.write(
        "\n"
    )


print(
    f"{len(packages)} package(s)"
)

PY


# ==============================================================================
# Validation catalogue
# ==============================================================================

CATALOG_COUNT="$(
    jq \
        '.count' \
        "${CATALOG_TMP}"
)"


REAL_PACKAGE_COUNT="$(
    find \
        "${PACKAGES}" \
        -maxdepth 1 \
        -type f \
        -name '*.json' \
        | wc -l
)"


[[ "${CATALOG_COUNT}" == "${REAL_PACKAGE_COUNT}" ]] \
    || {
        echo \
            "catalog.count=${CATALOG_COUNT}, packages=${REAL_PACKAGE_COUNT}"

        exit 1
    }


# Vérification des packages référencés.
python3 - \
    "${CATALOG_TMP}" \
    "${PACKAGES}" <<'PY'

import json
import pathlib
import sys


catalog_file = pathlib.Path(
    sys.argv[1]
)

packages_dir = pathlib.Path(
    sys.argv[2]
)


with catalog_file.open(
    "r",
    encoding="utf-8",
) as handle:

    catalog = json.load(
        handle
    )


for package in catalog.get(
    "packages",
    [],
):

    file_name = package[
        "file"
    ]

    checksum_name = package[
        "checksumFile"
    ]

    package_file = (
        packages_dir
        / file_name
    )

    checksum_file = (
        packages_dir
        / checksum_name
    )

    if not package_file.is_file():

        raise SystemExit(
            f"Package absent : {package_file}"
        )

    if not checksum_file.is_file():

        raise SystemExit(
            f"Checksum absent : {checksum_file}"
        )


print(
    "Références catalogue : OK"
)

PY


# ==============================================================================
# Catalogue publié EN DERNIER
# ==============================================================================

mv \
    "${CATALOG_TMP}" \
    "${CATALOG_FINAL}"


# ==============================================================================
# Résultat
# ==============================================================================

echo
echo "================================================================"
echo "                     NEXTLIMIT EUROPE"
echo "================================================================"
echo

echo \
    "Détectés     : ${TOTAL}"

echo \
    "Construits   : ${BUILT}"

echo \
    "Déjà présents: ${SKIPPED}"

echo \
    "Erreurs      : ${ERRORS}"

echo

echo \
    "Catalogue :"

echo \
    "  ${CATALOG_FINAL}"

echo

echo \
    "Packages :"

echo \
    "  ${PACKAGES}"

echo

echo \
    "Taille totale publiée :"

du \
    -sh \
    "${PACKAGES}"

echo

echo \
    "Rapport :"

echo \
    "  ${SUMMARY}"

echo


success \
    "Compilation Europe terminée."

CONTAINER_SCRIPT


# ==============================================================================
# Vérification côté hôte
# ==============================================================================

if [[ "${LIST_ONLY:-0}" == "1" ]]
then
    success "Inventaire terminé."
    exit 0
fi

CATALOG="${PUBLIC_ROOT}/catalog.json"


[[ -s "${CATALOG}" ]] \
    || die "catalog.json n'a pas été généré."


success \
    "Pipeline terminé."


echo

echo \
    "Catalogue :"

echo \
    "  ${CATALOG}"

echo

echo \
    "Packages :"

echo \
    "  ${PUBLIC_ROOT}/packages/"

echo


du \
    -sh \
    "${PUBLIC_ROOT}/packages"

echo