#!/usr/bin/env bash

set -Eeuo pipefail

# ==============================================================================
# NextLimit
# Build automatique des datasets SQLite v2 préindexés pour l'Europe
#
# Flux :
#
# Geofabrik .osm.pbf
#       ↓
# osm-dataset/build_dataset.py
#       ↓
# SQLite NextLimit v2
#       ↓
# graphe + RTree + indexes
#       ↓
# validations serveur
#       ↓
# gzip reproductible (niveau 9, mtime=0)
#       ↓
# SHA-256 du .sqlite.gz
#       ↓
# packages/{ID}-{VERSION}.sqlite.gz
# packages/{ID}-{VERSION}.sqlite.gz.sha256
# packages/{ID}.json
#       ↓
# catalog.json v2 publié en dernier
#
# Dépendance hôte :
#
#   Docker uniquement
#
# Arborescence attendue :
#
# /var/www/dl/osm/
# ├── build-europe-preindexed.sh
# └── osm-dataset/
#     ├── build_dataset.py
#     ├── dataset_schema.py
#     ├── publish_dataset.py
#     ├── requirements.txt
#     └── ...
#
# Usage :
#
#   ./build-europe-preindexed.sh
#
# Tests ciblés :
#
#   COUNTRIES="AD" ./build-europe-preindexed.sh
#   COUNTRIES="LU,BE" ./build-europe-preindexed.sh
#   COUNTRIES="FR" ./build-europe-preindexed.sh
#
# Forcer :
#
#   FORCE=1 COUNTRIES="FR" ./build-europe-preindexed.sh
#
# Conserver les PBF :
#
#   KEEP_SOURCE=1 ./build-europe-preindexed.sh
#
# Conserver les SQLite :
#
#   KEEP_SQLITE=1 ./build-europe-preindexed.sh
#
# ==============================================================================


# ==============================================================================
# Configuration hôte
# ==============================================================================

SCRIPT_DIR="$(
    cd "$(dirname "${BASH_SOURCE[0]}")"
    pwd
)"

PROJECT_DIR="${PROJECT_DIR:-${SCRIPT_DIR}}"

OSM_DATASET_DIR="${PROJECT_DIR}/osm-dataset"

BUILD_ROOT="${BUILD_ROOT:-${SCRIPT_DIR}/build/osm-europe}"

PUBLIC_ROOT="${PUBLIC_ROOT:-${SCRIPT_DIR}/public-osm/data}"

BASE_URL="${BASE_URL:-https://technique.prod1.parti.tech/osm/data/}"

IMAGE_NAME="${IMAGE_NAME:-nextlimit-osm-builder:latest}"

INDEX_URL="${INDEX_URL:-https://download.geofabrik.de/index-v1-nogeom.json}"

COUNTRIES="${COUNTRIES:-}"

FORCE="${FORCE:-0}"

KEEP_SOURCE="${KEEP_SOURCE:-0}"

KEEP_SQLITE="${KEEP_SQLITE:-0}"

MAX_PACKAGES="${MAX_PACKAGES:-0}"


# ==============================================================================
# Helpers hôte
# ==============================================================================

log()
{
    printf \
        '\n\033[1;34m[NextLimit]\033[0m %s\n' \
        "$*"
}


success()
{
    printf \
        '\n\033[1;32m[NextLimit]\033[0m %s\n' \
        "$*"
}


warning()
{
    printf \
        '\n\033[1;33m[NextLimit]\033[0m %s\n' \
        "$*"
}


die()
{
    printf \
        '\n\033[1;31m[ERREUR]\033[0m %s\n' \
        "$*" >&2

    exit 1
}


# ==============================================================================
# Vérifications hôte
# ==============================================================================

command -v docker >/dev/null 2>&1 \
    || die "Docker n'est pas installé."

docker info >/dev/null 2>&1 \
    || die "Docker n'est pas démarré ou inaccessible."

[[ -f "${OSM_DATASET_DIR}/build_dataset.py" ]] \
    || die "build_dataset.py introuvable : ${OSM_DATASET_DIR}"

[[ -f "${OSM_DATASET_DIR}/dataset_schema.py" ]] \
    || die "dataset_schema.py introuvable : ${OSM_DATASET_DIR}"

[[ -f "${OSM_DATASET_DIR}/requirements.txt" ]] \
    || die "requirements.txt introuvable : ${OSM_DATASET_DIR}"

mkdir -p \
    "${BUILD_ROOT}" \
    "${PUBLIC_ROOT}"


# ==============================================================================
# Image Docker
# ==============================================================================

log "Construction de l'image Docker de compilation..."


docker build \
    --tag "${IMAGE_NAME}" \
    - <<'DOCKERFILE'

FROM python:3.12-bookworm

ENV \
    DEBIAN_FRONTEND=noninteractive \
    PYTHONUNBUFFERED=1 \
    PIP_DISABLE_PIP_VERSION_CHECK=1

RUN apt-get update \
    && apt-get install -y --no-install-recommends \
        ca-certificates \
        curl \
        gzip \
        sqlite3 \
        osmium-tool \
        coreutils \
        jq \
        build-essential \
        cmake \
        libexpat1-dev \
        zlib1g-dev \
        libbz2-dev \
        libsqlite3-dev \
    && rm -rf /var/lib/apt/lists/*

WORKDIR /project

CMD ["bash"]

DOCKERFILE


# ==============================================================================
# Pipeline Docker
# ==============================================================================

log "Lancement du pipeline Europe..."


docker run \
    --rm \
    --interactive \
    --user "$(id -u):$(id -g)" \
    --env "INDEX_URL=${INDEX_URL}" \
    --env "BASE_URL=${BASE_URL}" \
    --env "COUNTRIES=${COUNTRIES}" \
    --env "FORCE=${FORCE}" \
    --env "KEEP_SOURCE=${KEEP_SOURCE}" \
    --env "KEEP_SQLITE=${KEEP_SQLITE}" \
    --env "MAX_PACKAGES=${MAX_PACKAGES}" \
    --volume "${PROJECT_DIR}:/project" \
    --volume "${BUILD_ROOT}:/build-osm" \
    --volume "${PUBLIC_ROOT}:/public-osm" \
    "${IMAGE_NAME}" \
    bash -s <<'CONTAINER_SCRIPT'

set -Eeuo pipefail


# ==============================================================================
# Répertoires conteneur
# ==============================================================================

PROJECT="/project"

TOOLS="${PROJECT}/osm-dataset"

BUILD="/build-osm"

SOURCES="${BUILD}/sources"

SQLITES="${BUILD}/sqlite"

REPORTS="${BUILD}/reports"

TMP="${BUILD}/tmp"

VENV="${BUILD}/venv"

PUBLIC="/public-osm"

PACKAGES="${PUBLIC}/packages"

INDEX_JSON="${BUILD}/geofabrik-index.json"

COUNTRY_LIST="${BUILD}/europe-packages.tsv"

SUMMARY="${BUILD}/build-summary.csv"

FAILED="${BUILD}/failed.tsv"


mkdir -p \
    "${SOURCES}" \
    "${SQLITES}" \
    "${REPORTS}" \
    "${TMP}" \
    "${PACKAGES}"


: > "${FAILED}"


# ==============================================================================
# Helpers conteneur
# ==============================================================================

log()
{
    printf \
        '\n\033[1;34m[Builder]\033[0m %s\n' \
        "$*"
}


success()
{
    printf \
        '\n\033[1;32m[Builder]\033[0m %s\n' \
        "$*"
}


warning()
{
    printf \
        '\n\033[1;33m[Builder]\033[0m %s\n' \
        "$*"
}


human_size()
{
    numfmt \
        --to=iec-i \
        --suffix=B \
        "$1"
}


# ==============================================================================
# Virtualenv
# ==============================================================================

log "Préparation du virtualenv..."


if [[ ! -x "${VENV}/bin/python" ]]
then

    python3 -m venv "${VENV}"

fi


"${VENV}/bin/python" \
    -m pip install \
    --upgrade \
    pip


"${VENV}/bin/python" \
    -m pip install \
    --requirement "${TOOLS}/requirements.txt"


# ==============================================================================
# Versions
# ==============================================================================

log "Versions utilisées"

echo

"${VENV}/bin/python" --version

osmium --version

sqlite3 --version

gzip --version | head -1

echo


# ==============================================================================
# Vérification RTree
# ==============================================================================

log "Vérification SQLite RTree..."


RTREE_TEST="$(
    sqlite3 ':memory:' \
        "CREATE VIRTUAL TABLE test_rtree USING rtree(id,minX,maxX,minY,maxY); SELECT rtreecheck('test_rtree');"
)"


[[ "${RTREE_TEST}" == "ok" ]] \
    || {
        echo "RTree SQLite indisponible."
        exit 1
    }


# ==============================================================================
# Index Geofabrik
# ==============================================================================

log "Téléchargement de l'index Geofabrik..."


curl \
    --fail \
    --location \
    --retry 10 \
    --retry-delay 5 \
    --output "${INDEX_JSON}" \
    "${INDEX_URL}"


# ==============================================================================
# Détection des pays européens
# ==============================================================================

log "Détection des packages Europe..."


"${VENV}/bin/python" - \
    "${INDEX_JSON}" \
    "${COUNTRY_LIST}" \
    "${COUNTRIES}" <<'PY'

import json
import sys


index_file = sys.argv[1]
output_file = sys.argv[2]
requested_raw = sys.argv[3].strip()


requested = {
    item.strip().upper()
    for item in requested_raw.split(",")
    if item.strip()
}


with open(
    index_file,
    "r",
    encoding="utf-8",
) as handle:

    payload = json.load(handle)


features = payload.get(
    "features",
    [],
)


by_id = {}


for feature in features:

    properties = feature.get(
        "properties",
        {},
    )

    feature_id = properties.get(
        "id"
    )

    if feature_id:

        by_id[feature_id] = properties


def belongs_to_europe(properties):

    feature_id = properties.get(
        "id",
        "",
    )

    if feature_id == "europe":

        return True

    parent = properties.get(
        "parent"
    )

    visited = set()

    while parent:

        if parent in visited:

            break

        visited.add(parent)

        if parent == "europe":

            return True

        parent_properties = by_id.get(
            parent
        )

        if parent_properties is None:

            break

        parent = parent_properties.get(
            "parent"
        )

    return False


rows = []


for properties in by_id.values():

    if not belongs_to_europe(
        properties
    ):

        continue


    iso_codes = (
        properties.get(
            "iso3166-1:alpha2"
        )
        or []
    )


    if not iso_codes:

        continue


    iso_codes = [
        str(code).upper()
        for code in iso_codes
    ]


    if requested:

        if not requested.intersection(
            iso_codes
        ):

            continue


    urls = properties.get(
        "urls"
    ) or {}


    pbf_url = urls.get(
        "pbf"
    )


    if not pbf_url:

        continue


    region_id = properties.get(
        "id"
    )

    name = (
        properties.get(
            "name"
        )
        or region_id
    )


    package_id = "-".join(
        sorted(
            iso_codes
        )
    )


    rows.append(
        (
            package_id,
            name,
            ",".join(
                iso_codes
            ),
            region_id,
            pbf_url,
        )
    )


rows.sort(
    key=lambda value: value[0]
)


with open(
    output_file,
    "w",
    encoding="utf-8",
) as handle:

    for row in rows:

        handle.write(
            "\t".join(
                row
            )
            + "\n"
        )


print(
    f"{len(rows)} package(s) détecté(s)"
)


for row in rows:

    print(
        f"{row[0]:10s} {row[1]}"
    )

PY


# ==============================================================================
# Rapport global
# ==============================================================================

echo \
"id,name,version,source_bytes,sqlite_bytes,gzip_bytes,segments,build_seconds,publish_seconds,status" \
> "${SUMMARY}"


TOTAL=0
BUILT=0
SKIPPED=0
ERRORS=0


# ==============================================================================
# Traitement pays par pays
# ==============================================================================

while IFS=$'\t' read -r \
    PACKAGE_ID \
    PACKAGE_NAME \
    ISO_CODES \
    GEOFABRIK_REGION \
    PBF_URL

do

    [[ -z "${PACKAGE_ID}" ]] && continue


    TOTAL=$((TOTAL + 1))


    if [[ "${MAX_PACKAGES:-0}" -gt 0 ]] \
        && [[ "${TOTAL}" -gt "${MAX_PACKAGES}" ]]
    then

        break

    fi


    echo
    echo "================================================================"
    echo "${PACKAGE_ID} - ${PACKAGE_NAME}"
    echo "================================================================"


    # ==========================================================================
    # Contexte réglementaire
    # ==========================================================================

    IFS=',' read -ra ISO_ARRAY <<< "${ISO_CODES}"


    if [[ "${#ISO_ARRAY[@]}" -eq 1 ]]
    then

        COUNTRY_CODE="${ISO_ARRAY[0]}"

    else

        COUNTRY_CODE="XX"

    fi


    SOURCE_FILE="${SOURCES}/${PACKAGE_ID}-latest.osm.pbf"


    # ==========================================================================
    # Téléchargement
    # ==========================================================================

    log "${PACKAGE_ID}: téléchargement Geofabrik"


    curl \
        --fail \
        --location \
        --retry 10 \
        --retry-delay 5 \
        --retry-all-errors \
        --continue-at - \
        --output "${SOURCE_FILE}" \
        "${PBF_URL}"


    osmium fileinfo \
        "${SOURCE_FILE}" \
        >/dev/null


    SOURCE_BYTES="$(
        stat -c%s "${SOURCE_FILE}"
    )"


    # ==========================================================================
    # Timestamp OSM
    # ==========================================================================

    SOURCE_TIMESTAMP="$(
        osmium fileinfo \
            -g header.option.osmosis_replication_timestamp \
            "${SOURCE_FILE}" \
            2>/dev/null \
            || true
    )"


    if [[ -z "${SOURCE_TIMESTAMP}" ]]
    then

        SOURCE_TIMESTAMP="$(
            osmium fileinfo \
                -g header.option.timestamp \
                "${SOURCE_FILE}" \
                2>/dev/null \
                || true
        )"

    fi


    if [[ -z "${SOURCE_TIMESTAMP}" ]]
    then

        warning "${PACKAGE_ID}: timestamp PBF absent."

        SOURCE_TIMESTAMP="$(
            date \
                -u \
                '+%Y-%m-%dT00:00:00Z'
        )"

    fi


    # ==========================================================================
    # Version YYYY.MM.DD
    # ==========================================================================

    VERSION="$(
        "${VENV}/bin/python" \
            - "${SOURCE_TIMESTAMP}" <<'PY'

import datetime
import sys


value = sys.argv[1]


try:

    value = value.replace(
        "Z",
        "+00:00",
    )

    parsed = datetime.datetime.fromisoformat(
        value
    )

    print(
        parsed.strftime(
            "%Y.%m.%d"
        )
    )

except Exception:

    print(
        datetime.datetime.now(
            datetime.timezone.utc
        ).strftime(
            "%Y.%m.%d"
        )
    )

PY
    )"


    DATASET_ID="${PACKAGE_ID}-${VERSION}"

    SQLITE_FILE="${SQLITES}/${PACKAGE_ID}.sqlite"

    REPORT_FILE="${REPORTS}/${PACKAGE_ID}-${VERSION}.report.json"

    PACKAGE_FILE="${PACKAGE_ID}-${VERSION}.sqlite.gz"

    CHECKSUM_FILE="${PACKAGE_FILE}.sha256"

    MANIFEST_FILE="${PACKAGE_ID}.json"

    EXPECTED_PACKAGE="${PACKAGES}/${PACKAGE_FILE}"

    EXPECTED_SHA="${PACKAGES}/${CHECKSUM_FILE}"

    EXPECTED_MANIFEST="${PACKAGES}/${MANIFEST_FILE}"


    # ==========================================================================
    # Skip package existant
    # ==========================================================================

    if [[ "${FORCE:-0}" != "1" ]] \
        && [[ -s "${EXPECTED_PACKAGE}" ]] \
        && [[ -s "${EXPECTED_SHA}" ]] \
        && [[ -s "${EXPECTED_MANIFEST}" ]]

    then

        if (
            cd "${PACKAGES}"

            sha256sum \
                -c \
                "${CHECKSUM_FILE}" \
                >/dev/null
        )
        then

            success \
                "${PACKAGE_ID} ${VERSION} déjà publié."

            SKIPPED=$((SKIPPED + 1))


            if [[ "${KEEP_SOURCE:-0}" != "1" ]]
            then

                rm -f \
                    "${SOURCE_FILE}"

            fi


            continue

        fi

    fi


    # ==========================================================================
    # Build SQLite v2
    # ==========================================================================

    rm -f \
        "${SQLITE_FILE}"


    BUILD_STARTED="$(
        date +%s
    )"


    log \
        "${PACKAGE_ID}: génération SQLite v2"


    "${VENV}/bin/python" \
        "${TOOLS}/build_dataset.py" \
        "${SOURCE_FILE}" \
        "${SQLITE_FILE}" \
        --dataset-id "${DATASET_ID}" \
        --country "${COUNTRY_CODE}" \
        --region "${PACKAGE_ID}" \
        --source-date "${SOURCE_TIMESTAMP}" \
        --report "${REPORT_FILE}" \
        --force


    BUILD_ENDED="$(
        date +%s
    )"


    BUILD_SECONDS=$((BUILD_ENDED - BUILD_STARTED))


    SQLITE_BYTES="$(
        stat -c%s "${SQLITE_FILE}"
    )"


    # ==========================================================================
    # Validation SQLite
    # ==========================================================================

    log \
        "${PACKAGE_ID}: validation SQLite"


    QUICK_CHECK="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "PRAGMA quick_check(1);"
    )"


    [[ "${QUICK_CHECK}" == "ok" ]] \
        || {
            echo \
                "PRAGMA quick_check a échoué."

            exit 1
        }


    SCHEMA_VERSION="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT schema_version FROM dataset_metadata LIMIT 1;"
    )"


    [[ "${SCHEMA_VERSION}" == "2" ]] \
        || {
            echo \
                "schema_version=${SCHEMA_VERSION}, attendu=2"

            exit 1
        }


    REGION_CODE="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT region_code FROM dataset_metadata LIMIT 1;"
    )"


    [[ "${REGION_CODE}" == "${PACKAGE_ID}" ]] \
        || {
            echo \
                "region_code incorrect : ${REGION_CODE}"

            exit 1
        }


    META_SEGMENTS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT segment_count FROM dataset_metadata LIMIT 1;"
    )"


    ROAD_SEGMENTS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT COUNT(*) FROM road_segments;"
    )"


    RTREE_SEGMENTS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT COUNT(*) FROM road_segment_rtree;"
    )"


    [[ "${META_SEGMENTS}" -gt 0 ]] \
        || {
            echo \
                "segment_count doit être > 0."

            exit 1
        }


    [[ "${META_SEGMENTS}" == "${ROAD_SEGMENTS}" ]] \
        || {
            echo \
                "segment_count != road_segments (${META_SEGMENTS}/${ROAD_SEGMENTS})."

            exit 1
        }


    [[ "${META_SEGMENTS}" == "${RTREE_SEGMENTS}" ]] \
        || {
            echo \
                "segment_count != road_segment_rtree (${META_SEGMENTS}/${RTREE_SEGMENTS})."

            exit 1
        }


    RTREE_CHECK="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT rtreecheck('road_segment_rtree');"
    )"


    [[ "${RTREE_CHECK}" == "ok" ]] \
        || {
            echo \
                "rtreecheck a échoué : ${RTREE_CHECK}"

            exit 1
        }


    for REQUIRED_INDEX in \
        road_segments_from_node_idx \
        road_segments_to_node_idx \
        road_segments_way_idx

    do

        INDEX_COUNT="$(
            sqlite3 \
                "${SQLITE_FILE}" \
                "
                SELECT COUNT(*)
                FROM sqlite_master
                WHERE
                    type = 'index'
                    AND name = '${REQUIRED_INDEX}';
                "
        )"


        [[ "${INDEX_COUNT}" == "1" ]] \
            || {
                echo \
                    "Index manquant : ${REQUIRED_INDEX}"

                exit 1
            }

    done


    MISSING_RTREE="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "
            SELECT COUNT(*)
            FROM road_segments s
            LEFT JOIN road_segment_rtree r
                ON r.row_id = s.row_id
            WHERE r.row_id IS NULL;
            "
    )"


    [[ "${MISSING_RTREE}" == "0" ]] \
        || {
            echo \
                "${MISSING_RTREE} segment(s) sans RTree."

            exit 1
        }


    INVALID_SPEEDS="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "
            SELECT COUNT(*)
            FROM road_segments
            WHERE
                (
                    speed_limit_kmh IS NOT NULL
                    AND (
                        speed_limit_kmh < 5
                        OR speed_limit_kmh > 250
                    )
                )
                OR speed_source < 0
                OR speed_source > 4
                OR speed_confidence < 0
                OR speed_confidence > 100;
            "
    )"


    [[ "${INVALID_SPEEDS}" == "0" ]] \
        || {
            echo \
                "${INVALID_SPEEDS} limitation(s) incohérente(s)."

            exit 1
        }


    OSM_ATTRIBUTION="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT osm_attribution FROM dataset_metadata LIMIT 1;"
    )"


    OSM_LICENSE="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT osm_license FROM dataset_metadata LIMIT 1;"
    )"


    [[ -n "${OSM_ATTRIBUTION}" ]] \
        || {
            echo \
                "Attribution OSM absente."

            exit 1
        }


    [[ "${OSM_LICENSE}" == "ODbL-1.0" ]] \
        || {
            echo \
                "Licence OSM incorrecte : ${OSM_LICENSE}"

            exit 1
        }


    # ==========================================================================
    # Publication NextLimit v2
    # ==========================================================================

    PUBLISH_STAGE="$(
        mktemp \
            -d \
            "${TMP}/${PACKAGE_ID}.publish.XXXXXX"
    )"


    # Nettoyage même en cas d'erreur dans la phase suivante.
    cleanup_publish_stage()
    {
        if [[ -n "${PUBLISH_STAGE:-}" ]] \
            && [[ -d "${PUBLISH_STAGE}" ]]
        then

            rm -rf \
                "${PUBLISH_STAGE}"

        fi
    }


    STAGED_PACKAGE="${PUBLISH_STAGE}/${PACKAGE_FILE}"

    STAGED_SHA="${PUBLISH_STAGE}/${CHECKSUM_FILE}"

    STAGED_MANIFEST="${PUBLISH_STAGE}/${MANIFEST_FILE}"


    PUBLISH_STARTED="$(
        date +%s
    )"


    # ==========================================================================
    # Gzip reproductible
    # ==========================================================================

    log \
        "${PACKAGE_ID}: compression gzip reproductible"


    "${VENV}/bin/python" - \
        "${SQLITE_FILE}" \
        "${STAGED_PACKAGE}" <<'PY'

import gzip
import shutil
import sys


source = sys.argv[1]
destination = sys.argv[2]


with open(
    source,
    "rb",
) as src:

    with open(
        destination,
        "wb",
    ) as raw_dst:

        with gzip.GzipFile(
            filename="",
            mode="wb",
            fileobj=raw_dst,
            compresslevel=9,
            mtime=0,
        ) as dst:

            shutil.copyfileobj(
                src,
                dst,
                length=1024 * 1024,
            )

PY


    [[ -s "${STAGED_PACKAGE}" ]] \
        || {
            cleanup_publish_stage

            echo \
                "Le gzip n'a pas été généré."

            exit 1
        }


    PUBLISH_ENDED="$(
        date +%s
    )"


    PUBLISH_SECONDS=$((PUBLISH_ENDED - PUBLISH_STARTED))


    GZIP_BYTES="$(
        stat -c%s "${STAGED_PACKAGE}"
    )"


    # ==========================================================================
    # Validation gzip
    # ==========================================================================

    log \
        "${PACKAGE_ID}: validation gzip"


    gzip \
        --test \
        "${STAGED_PACKAGE}"


    ORIGINAL_SQLITE_SHA="$(
        sha256sum \
            "${SQLITE_FILE}" \
        | awk \
            '{print $1}'
    )"


    DECOMPRESSED_SQLITE_SHA="$(
        gzip \
            -dc \
            "${STAGED_PACKAGE}" \
        | sha256sum \
        | awk \
            '{print $1}'
    )"


    [[ "${ORIGINAL_SQLITE_SHA}" == "${DECOMPRESSED_SQLITE_SHA}" ]] \
        || {
            cleanup_publish_stage

            echo \
                "Le gzip ne restitue pas exactement le SQLite source."

            exit 1
        }


    # ==========================================================================
    # SHA-256 du gzip
    # ==========================================================================

    (
        cd "${PUBLISH_STAGE}"

        sha256sum \
            "${PACKAGE_FILE}" \
            > "${CHECKSUM_FILE}"
    )


    [[ -s "${STAGED_SHA}" ]] \
        || {
            cleanup_publish_stage

            echo \
                "Checksum non généré."

            exit 1
        }


    (
        cd "${PUBLISH_STAGE}"

        sha256sum \
            -c \
            "${CHECKSUM_FILE}"
    )


    # ==========================================================================
    # Metadata SQLite
    # ==========================================================================

    GENERATED_AT="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT generated_at FROM dataset_metadata LIMIT 1;"
    )"


    DB_SOURCE_TIMESTAMP="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT source_timestamp FROM dataset_metadata LIMIT 1;"
    )"


    SEGMENT_COUNT="$(
        sqlite3 \
            "${SQLITE_FILE}" \
            "SELECT segment_count FROM dataset_metadata LIMIT 1;"
    )"


    # ==========================================================================
    # Manifest
    # ==========================================================================

    log \
        "${PACKAGE_ID}: génération du manifest"


    "${VENV}/bin/python" - \
        "${STAGED_MANIFEST}" \
        "${PACKAGE_ID}" \
        "${VERSION}" \
        "${PACKAGE_NAME}" \
        "${PACKAGE_FILE}" \
        "${CHECKSUM_FILE}" \
        "${GZIP_BYTES}" \
        "${SQLITE_BYTES}" \
        "${GENERATED_AT}" \
        "${DB_SOURCE_TIMESTAMP}" \
        "${SEGMENT_COUNT}" <<'PY'

import json
import sys


(
    output,
    package_id,
    version,
    name,
    package_file,
    checksum_file,
    gzip_size,
    installed_size,
    generated_at,
    source_timestamp,
    segment_count,
) = sys.argv[1:]


manifest = {

    "id": package_id,

    "name": name,

    "version": version,

    "format": "nextlimit_sqlite",

    "compression": "gzip",

    "schemaVersion": 2,

    "file": package_file,

    "checksumFile": checksum_file,

    "size": int(
        gzip_size
    ),

    "installedSize": int(
        installed_size
    ),

    "generatedAt": generated_at,

    "sourceTimestamp": source_timestamp,

    "segmentCount": int(
        segment_count
    ),
}


with open(
    output,
    "w",
    encoding="utf-8",
) as handle:

    json.dump(
        manifest,
        handle,
        ensure_ascii=False,
        indent=2,
    )

    handle.write(
        "\n"
    )

PY


    # ==========================================================================
    # Validation manifest
    # ==========================================================================

    "${VENV}/bin/python" - \
        "${STAGED_MANIFEST}" \
        "${PACKAGE_ID}" \
        "${VERSION}" \
        "${GZIP_BYTES}" \
        "${SQLITE_BYTES}" \
        "${SEGMENT_COUNT}" <<'PY'

import json
import sys


(
    manifest_file,
    package_id,
    version,
    gzip_size,
    sqlite_size,
    segment_count,
) = sys.argv[1:]


with open(
    manifest_file,
    "r",
    encoding="utf-8",
) as handle:

    data = json.load(
        handle
    )


expected = {

    "id": package_id,

    "version": version,

    "format": "nextlimit_sqlite",

    "compression": "gzip",

    "schemaVersion": 2,
}


for key, expected_value in expected.items():

    actual = data.get(
        key
    )

    if actual != expected_value:

        raise SystemExit(
            f"{key}: "
            f"{actual!r} != "
            f"{expected_value!r}"
        )


if int(
    data["size"]
) != int(
    gzip_size
):

    raise SystemExit(
        "size incorrect"
    )


if int(
    data["installedSize"]
) != int(
    sqlite_size
):

    raise SystemExit(
        "installedSize incorrect"
    )


if int(
    data["segmentCount"]
) != int(
    segment_count
):

    raise SystemExit(
        "segmentCount incorrect"
    )


if not data["file"].endswith(
    ".sqlite.gz"
):

    raise SystemExit(
        "Extension package invalide"
    )

PY


    # ==========================================================================
    # Publication atomique
    #
    # IMPORTANT :
    #
    # Le PUBLISH_STAGE n'est utilisé QU'ICI.
    #
    # Après ces mv, aucun ancien bloc ne doit essayer d'y relire quoi que ce soit.
    # ==========================================================================

    log \
        "${PACKAGE_ID}: publication atomique"


    mv \
        "${STAGED_PACKAGE}" \
        "${PACKAGES}/${PACKAGE_FILE}"


    mv \
        "${STAGED_SHA}" \
        "${PACKAGES}/${CHECKSUM_FILE}"


    mv \
        "${STAGED_MANIFEST}" \
        "${PACKAGES}/${MANIFEST_FILE}"


    cleanup_publish_stage


    success \
        "${PACKAGE_ID}: package publié."


    # ==========================================================================
    # Rapport
    # ==========================================================================

    printf \
        '"%s","%s","%s",%s,%s,%s,%s,%s,%s,"OK"\n' \
        "${PACKAGE_ID}" \
        "${PACKAGE_NAME//\"/\"\"}" \
        "${VERSION}" \
        "${SOURCE_BYTES}" \
        "${SQLITE_BYTES}" \
        "${GZIP_BYTES}" \
        "${META_SEGMENTS}" \
        "${BUILD_SECONDS}" \
        "${PUBLISH_SECONDS}" \
        >> "${SUMMARY}"


    echo

    echo \
        "Source PBF : $(human_size "${SOURCE_BYTES}")"

    echo \
        "SQLite     : $(human_size "${SQLITE_BYTES}")"

    echo \
        "Gzip       : $(human_size "${GZIP_BYTES}")"

    echo \
        "Segments   : ${META_SEGMENTS}"

    echo \
        "Build      : ${BUILD_SECONDS} s"

    echo \
        "Compression: ${PUBLISH_SECONDS} s"


    # ==========================================================================
    # Nettoyage
    # ==========================================================================

    if [[ "${KEEP_SOURCE:-0}" != "1" ]]
    then

        rm -f \
            "${SOURCE_FILE}"

    fi


    if [[ "${KEEP_SQLITE:-0}" != "1" ]]
    then

        rm -f \
            "${SQLITE_FILE}"

    fi


    BUILT=$((BUILT + 1))


    success \
        "${PACKAGE_ID} ${VERSION} terminé."


done < "${COUNTRY_LIST}"


# ==============================================================================
# Catalogue public v2
# ==============================================================================

log \
    "Génération catalog.json v2..."


CATALOG_TMP="${PUBLIC}/catalog.json.tmp"

CATALOG_FINAL="${PUBLIC}/catalog.json"


"${VENV}/bin/python" - \
    "${PACKAGES}" \
    "${COUNTRY_LIST}" \
    "${CATALOG_TMP}" <<'PY'

import datetime
import json
import pathlib
import sys


packages_dir = pathlib.Path(
    sys.argv[1]
)

country_list = pathlib.Path(
    sys.argv[2]
)

output = pathlib.Path(
    sys.argv[3]
)


country_metadata = {}


with country_list.open(
    "r",
    encoding="utf-8",
) as handle:

    for line in handle:

        line = line.rstrip(
            "\n"
        )

        if not line:

            continue

        (
            package_id,
            name,
            iso_codes,
            region_id,
            _source_url,
        ) = line.split(
            "\t"
        )

        country_metadata[
            package_id
        ] = {

            "name": name,

            "iso3166_1": iso_codes.split(
                ","
            ),

            "geofabrikRegionId": region_id,
        }


packages = []


for manifest_path in sorted(
    packages_dir.glob(
        "*.json"
    )
):

    with manifest_path.open(
        "r",
        encoding="utf-8",
    ) as handle:

        manifest = json.load(
            handle
        )


    package_id = manifest[
        "id"
    ]


    metadata = country_metadata.get(
        package_id,
        {},
    )


    package = {

        "id": package_id,

        "name": metadata.get(
            "name",
            manifest.get(
                "name",
                package_id,
            ),
        ),

        "iso3166_1": metadata.get(
            "iso3166_1",
            [],
        ),

        "geofabrikRegionId": metadata.get(
            "geofabrikRegionId"
        ),

        "version": manifest[
            "version"
        ],

        "format": manifest[
            "format"
        ],

        "compression": manifest[
            "compression"
        ],

        "schemaVersion": int(
            manifest[
                "schemaVersion"
            ]
        ),

        "file": manifest[
            "file"
        ],

        "checksumFile": manifest[
            "checksumFile"
        ],

        "size": int(
            manifest[
                "size"
            ]
        ),

        "installedSize": int(
            manifest[
                "installedSize"
            ]
        ),

        "generatedAt": manifest[
            "generatedAt"
        ],
    }


    if manifest.get(
        "sourceTimestamp"
    ):

        package[
            "sourceTimestamp"
        ] = manifest[
            "sourceTimestamp"
        ]


    if manifest.get(
        "segmentCount"
    ) is not None:

        package[
            "segmentCount"
        ] = int(
            manifest[
                "segmentCount"
            ]
        )


    packages.append(
        package
    )


catalog = {

    "version": 2,

    "region": "europe",

    "generatedAt": (
        datetime.datetime.now(
            datetime.timezone.utc
        )
        .replace(
            microsecond=0
        )
        .isoformat()
        .replace(
            "+00:00",
            "Z",
        )
    ),

    "count": len(
        packages
    ),

    "packages": packages,
}


serialized = json.dumps(
    catalog,
    ensure_ascii=False,
    indent=2,
)


# Aucun lien source Geofabrik dans le catalogue mobile.
if "download.geofabrik.de" in serialized:

    raise SystemExit(
        "URL Geofabrik détectée dans catalog.json"
    )


with output.open(
    "w",
    encoding="utf-8",
) as handle:

    handle.write(
        serialized
    )

    handle.write(
        "\n"
    )


print(
    f"{len(packages)} package(s)"
)

PY


# ==============================================================================
# Validation catalogue
# ==============================================================================

CATALOG_COUNT="$(
    jq \
        '.count' \
        "${CATALOG_TMP}"
)"


REAL_PACKAGE_COUNT="$(
    find \
        "${PACKAGES}" \
        -maxdepth 1 \
        -type f \
        -name '*.json' \
        | wc -l
)"


[[ "${CATALOG_COUNT}" == "${REAL_PACKAGE_COUNT}" ]] \
    || {
        echo \
            "catalog.count=${CATALOG_COUNT}, packages=${REAL_PACKAGE_COUNT}"

        exit 1
    }


# Vérification des packages référencés.
"${VENV}/bin/python" - \
    "${CATALOG_TMP}" \
    "${PACKAGES}" <<'PY'

import json
import pathlib
import sys


catalog_file = pathlib.Path(
    sys.argv[1]
)

packages_dir = pathlib.Path(
    sys.argv[2]
)


with catalog_file.open(
    "r",
    encoding="utf-8",
) as handle:

    catalog = json.load(
        handle
    )


for package in catalog.get(
    "packages",
    [],
):

    file_name = package[
        "file"
    ]

    checksum_name = package[
        "checksumFile"
    ]

    package_file = (
        packages_dir
        / file_name
    )

    checksum_file = (
        packages_dir
        / checksum_name
    )

    if not package_file.is_file():

        raise SystemExit(
            f"Package absent : {package_file}"
        )

    if not checksum_file.is_file():

        raise SystemExit(
            f"Checksum absent : {checksum_file}"
        )


print(
    "Références catalogue : OK"
)

PY


# ==============================================================================
# Catalogue publié EN DERNIER
# ==============================================================================

mv \
    "${CATALOG_TMP}" \
    "${CATALOG_FINAL}"


# ==============================================================================
# Résultat
# ==============================================================================

echo
echo "================================================================"
echo "                     NEXTLIMIT EUROPE"
echo "================================================================"
echo

echo \
    "Détectés     : ${TOTAL}"

echo \
    "Construits   : ${BUILT}"

echo \
    "Déjà présents: ${SKIPPED}"

echo \
    "Erreurs      : ${ERRORS}"

echo

echo \
    "Catalogue :"

echo \
    "  ${CATALOG_FINAL}"

echo

echo \
    "Packages :"

echo \
    "  ${PACKAGES}"

echo

echo \
    "Taille totale publiée :"

du \
    -sh \
    "${PACKAGES}"

echo

echo \
    "Rapport :"

echo \
    "  ${SUMMARY}"

echo


success \
    "Compilation Europe terminée."

CONTAINER_SCRIPT


# ==============================================================================
# Vérification côté hôte
# ==============================================================================

CATALOG="${PUBLIC_ROOT}/catalog.json"


[[ -s "${CATALOG}" ]] \
    || die "catalog.json n'a pas été généré."


success \
    "Pipeline terminé."


echo

echo \
    "Catalogue :"

echo \
    "  ${CATALOG}"

echo

echo \
    "Packages :"

echo \
    "  ${PUBLIC_ROOT}/packages/"

echo


du \
    -sh \
    "${PUBLIC_ROOT}/packages"

echo