diff -Nru osm2pgsql-1.8.1+ds/.clang-tidy osm2pgsql-1.9.0+ds/.clang-tidy --- osm2pgsql-1.8.1+ds/.clang-tidy 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.clang-tidy 2023-08-15 19:18:18.000000000 +0000 @@ -83,4 +83,8 @@ value: true - key: cppcoreguidelines-non-private-member-variables-in-classes.IgnoreClassesWithAllMemberVariablesBeingPublic value: true + - key: readability-function-cognitive-complexity.Threshold + value: 100 + - key: readability-function-cognitive-complexity.IgnoreMacros + value: true ... diff -Nru osm2pgsql-1.8.1+ds/CMakeLists.txt osm2pgsql-1.9.0+ds/CMakeLists.txt --- osm2pgsql-1.8.1+ds/CMakeLists.txt 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/CMakeLists.txt 2023-08-15 19:18:18.000000000 +0000 @@ -1,7 +1,7 @@ -cmake_minimum_required(VERSION 3.5.0) +cmake_minimum_required(VERSION 3.8.0) -project(osm2pgsql VERSION 1.8.1 LANGUAGES CXX C) +project(osm2pgsql VERSION 1.9.0 LANGUAGES CXX C) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -37,12 +37,8 @@ set(CMAKE_BUILD_TYPE RelWithDebInfo) endif() -if (NOT "${CMAKE_CXX_STANDARD}") - set(CMAKE_CXX_STANDARD 17) -endif() -message(STATUS "Building in C++${CMAKE_CXX_STANDARD} mode") +# We don't want to use special compiler extensions because we want portability set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_CXX_STANDARD_REQUIRED ON) if (MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS -DNOMINMAX) @@ -205,6 +201,14 @@ find_package(Threads) +find_path(NLOHMANN_INCLUDE_DIR nlohmann/json.hpp) +include_directories(SYSTEM ${NLOHMANN_INCLUDE_DIR}) + +find_path(POTRACE_INCLUDE_DIR potracelib.h) +find_library(POTRACE_LIBRARY NAMES potrace) + +find_path(CIMG_INCLUDE_DIR CImg.h) + ############### Libraries are found now ######################## set(LIBS ${Boost_LIBRARIES} ${PostgreSQL_LIBRARY} ${OSMIUM_LIBRARIES}) @@ -276,6 +280,33 @@ add_executable(osm2pgsql src/osm2pgsql.cpp) target_link_libraries(osm2pgsql osm2pgsql_lib ${LIBS}) +if (${POTRACE_LIBRARY} STREQUAL "POTRACE_LIBRARY-NOTFOUND" OR ${CIMG_INCLUDE_DIR} STREQUAL "CIMG_INCLUDE_DIR-NOTFOUND") + message(STATUS "Did not find cimg and/or potrace library. Not building osm2pgsql-gen.") +else() + if (WITH_LUA) + message(STATUS "Found cimg and potrace library. Building osm2pgsql-gen.") + set(BUILD_GEN 1) + include_directories(SYSTEM ${CIMG_INCLUDE_DIR}) + include_directories(SYSTEM ${POTRACE_INCLUDE_DIR}) + add_executable(osm2pgsql-gen src/gen/osm2pgsql-gen.cpp + src/gen/canvas.cpp + src/gen/gen-base.cpp + src/gen/gen-create.cpp + src/gen/gen-discrete-isolation.cpp + src/gen/gen-rivers.cpp + src/gen/gen-tile-builtup.cpp + src/gen/gen-tile-raster.cpp + src/gen/gen-tile-vector.cpp + src/gen/gen-tile.cpp + src/gen/params.cpp + src/gen/raster.cpp + src/gen/tracer.cpp) + target_link_libraries(osm2pgsql-gen osm2pgsql_lib ${LIBS} ${POTRACE_LIBRARY}) + else() + message(STATUS "No Lua. Not building osm2pgsql-gen.") + endif() +endif() + ############################################################# # Optional "clang-tidy" target ############################################################# @@ -287,7 +318,7 @@ if (CLANG_TIDY) message(STATUS "Looking for clang-tidy - found ${CLANG_TIDY}") - file(GLOB CT_CHECK_FILES src/*.cpp tests/*cpp) + file(GLOB CT_CHECK_FILES src/*.cpp src/*/*.cpp tests/*cpp) add_custom_target(clang-tidy ${CLANG_TIDY} @@ -333,4 +364,8 @@ install(TARGETS osm2pgsql DESTINATION bin) install(FILES default.style empty.style DESTINATION share/osm2pgsql) install(PROGRAMS scripts/osm2pgsql-replication DESTINATION bin) + if (BUILD_GEN) + install(TARGETS osm2pgsql-gen COMPONENT gen EXCLUDE_FROM_ALL DESTINATION bin) + add_custom_target(install-gen cmake --install ${CMAKE_BINARY_DIR} --component gen) + endif() endif() diff -Nru osm2pgsql-1.8.1+ds/CONTRIBUTING.md osm2pgsql-1.9.0+ds/CONTRIBUTING.md --- osm2pgsql-1.8.1+ds/CONTRIBUTING.md 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/CONTRIBUTING.md 2023-08-15 19:18:18.000000000 +0000 @@ -7,7 +7,7 @@ https://help.github.com/articles/using-pull-requests You should fork the project into your own repo, create a topic branch -there and then make one or more pull requests back to the openstreetmap repository. +there and then make one or more pull requests back to the OpenStreetMap repository. Your pull requests will then be reviewed and discussed. ## History @@ -79,6 +79,10 @@ ## Testing +osm2pgsql is tested with two types of tests: Classic tests written in C++ and BDD (Behavior Driven Development) tests written in Python. + +### Classic Tests + The code comes with a suite of tests. They are only compiled and run when `BUILD_TESTS=ON` is set in the CMake config. @@ -120,12 +124,7 @@ drops you into a shell after a failed test where you can still access the database created by `pg_virtualenv`. -### Performance testing - -If performance testing with a full planet import is required, indicate what -needs testing in a pull request. - -### BDD testing +### BDD Tests Tests in the `tests/bdd` directory use [behave](https://github.com/behave/behave), a Python implementation of a behaviour-driven test framework. To run the @@ -145,7 +144,7 @@ ``` Per default, behave assumes that the build directory is under `osm2pgsql/build`. -If your setup works like that, you can leave out the -D parameter. +If your setup works like that, you can leave out the `-D` parameter. To make this a bit easier a shell script `run-behave` is provided in your build directory which sets those correct paths and calls `behave`. If run @@ -178,6 +177,21 @@ proj support and skip tests accordingly. They also check for the test tablespace `tablespacetest` for tests that need tablespaces. +BDD tests hide print statements by default. For development purposes they +can be shown by adding these lines to `tests/bdd/.behaverc`: + +``` +color=False +stdout_capture=False +stderr_capture=False +log_capture=False +``` + +### Performance testing + +If performance testing with a full planet import is required, indicate what +needs testing in a pull request. + ## Coverage reports To create coverage reports, set `BUILD_COVERAGE` in the CMake config to `ON`, diff -Nru osm2pgsql-1.8.1+ds/debian/changelog osm2pgsql-1.9.0+ds/debian/changelog --- osm2pgsql-1.8.1+ds/debian/changelog 2023-07-13 17:10:14.000000000 +0000 +++ osm2pgsql-1.9.0+ds/debian/changelog 2023-08-16 04:25:14.000000000 +0000 @@ -1,3 +1,11 @@ +osm2pgsql (1.9.0+ds-1) unstable; urgency=medium + + * New upstream release. + * Drop gcc-13 patch, included upstream. + * Update build dependencies for osm2pgsql-gen. + + -- Bas Couwenberg Wed, 16 Aug 2023 06:25:14 +0200 + osm2pgsql (1.8.1+ds-2) unstable; urgency=medium * Bump debhelper compat to 13. diff -Nru osm2pgsql-1.8.1+ds/debian/control osm2pgsql-1.9.0+ds/debian/control --- osm2pgsql-1.8.1+ds/debian/control 2023-06-28 11:07:28.000000000 +0000 +++ osm2pgsql-1.9.0+ds/debian/control 2023-08-16 04:21:57.000000000 +0000 @@ -7,6 +7,7 @@ Section: utils Priority: optional Build-Depends: debhelper-compat (= 13), + cimg-dev, cmake, dh-python, libboost-dev, @@ -16,11 +17,13 @@ libexpat1-dev, libfmt-dev, libosmium2-dev (>= 2.17.3), + libpotrace-dev, libpq-dev, libproj-dev, zlib1g-dev, liblua5.3-dev, lua5.3, + nlohmann-json3-dev, python3, python3-argparse-manpage, python3-psycopg2, diff -Nru osm2pgsql-1.8.1+ds/debian/patches/0001-Add-missing-cstdint-header-includes-to-some-files.patch osm2pgsql-1.9.0+ds/debian/patches/0001-Add-missing-cstdint-header-includes-to-some-files.patch --- osm2pgsql-1.8.1+ds/debian/patches/0001-Add-missing-cstdint-header-includes-to-some-files.patch 2023-06-14 10:43:34.000000000 +0000 +++ osm2pgsql-1.9.0+ds/debian/patches/0001-Add-missing-cstdint-header-includes-to-some-files.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,86 +0,0 @@ -Description: Add missing cstdint header includes to some files -Author: Jochen Topf -Origin: https://github.com/openstreetmap/osm2pgsql/commit/c619302bc538e1dbf74dccf7f59d5b0d8f12ef5d -Bug: https://github.com/openstreetmap/osm2pgsql/issues/1957 -Bug-Debian: https://bugs.debian.org/1037812 - ---- a/src/expire-tiles.hpp -+++ b/src/expire-tiles.hpp -@@ -10,6 +10,7 @@ - * For a full list of authors see the git log. - */ - -+#include - #include - #include - #include ---- a/src/flex-index.hpp -+++ b/src/flex-index.hpp -@@ -11,6 +11,7 @@ - */ - - #include -+#include - #include - #include - #include ---- a/src/flex-write.cpp -+++ b/src/flex-write.cpp -@@ -16,6 +16,7 @@ - - #include - #include -+#include - #include - #include - #include ---- a/src/gazetteer-style.hpp -+++ b/src/gazetteer-style.hpp -@@ -10,6 +10,7 @@ - * For a full list of authors see the git log. - */ - -+#include - #include - #include - #include ---- a/src/middle-pgsql.cpp -+++ b/src/middle-pgsql.cpp -@@ -16,6 +16,7 @@ - */ - - #include -+#include - #include - #include - #include ---- a/src/pgsql-capabilities.hpp -+++ b/src/pgsql-capabilities.hpp -@@ -10,6 +10,7 @@ - * For a full list of authors see the git log. - */ - -+#include - #include - - class pg_conn_t; ---- a/src/pgsql.hpp -+++ b/src/pgsql.hpp -@@ -25,6 +25,7 @@ - #include - #include - #include -+#include - #include - #include - #include ---- a/src/progress-display.hpp -+++ b/src/progress-display.hpp -@@ -19,6 +19,7 @@ - */ - - #include -+#include - #include - - #include diff -Nru osm2pgsql-1.8.1+ds/debian/patches/series osm2pgsql-1.9.0+ds/debian/patches/series --- osm2pgsql-1.8.1+ds/debian/patches/series 2023-06-14 10:41:44.000000000 +0000 +++ osm2pgsql-1.9.0+ds/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -0001-Add-missing-cstdint-header-includes-to-some-files.patch diff -Nru osm2pgsql-1.8.1+ds/docs/CMakeLists.txt osm2pgsql-1.9.0+ds/docs/CMakeLists.txt --- osm2pgsql-1.8.1+ds/docs/CMakeLists.txt 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/docs/CMakeLists.txt 2023-08-15 19:18:18.000000000 +0000 @@ -25,6 +25,17 @@ VERBATIM) list(APPEND MANPAGE_TARGETS osm2pgsql.1) + + if(BUILD_GEN) + add_custom_command(OUTPUT osm2pgsql-gen.1 + COMMAND ${PANDOC} ${PANDOC_MAN_OPTIONS} -o osm2pgsql-gen.1 + ${CMAKE_CURRENT_SOURCE_DIR}/osm2pgsql-gen.md + DEPENDS osm2pgsql-gen.md manpage.template + COMMENT "Building manpage osm2pgsql-gen.1" + VERBATIM) + list(APPEND MANPAGE_TARGETS osm2pgsql-gen.1) + endif() + else() message(STATUS "Looking for pandoc - not found") message(STATUS " osm2pgsql manual page can not be built") @@ -53,4 +64,7 @@ if(ENABLE_INSTALL) install(FILES osm2pgsql.1 DESTINATION share/man/man1) install(FILES osm2pgsql-replication.1 DESTINATION share/man/man1) + if (BUILD_GEN) + install(FILES osm2pgsql-gen.1 COMPONENT gen EXCLUDE_FROM_ALL DESTINATION share/man/man1) + endif() endif() diff -Nru osm2pgsql-1.8.1+ds/docs/manpage_from_python.py osm2pgsql-1.9.0+ds/docs/manpage_from_python.py --- osm2pgsql-1.8.1+ds/docs/manpage_from_python.py 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/docs/manpage_from_python.py 2023-08-15 19:18:18.000000000 +0000 @@ -31,6 +31,7 @@ manpage = re.sub(r'.TH.*', f'.TH "{parser.prog.upper()}" "1" "{args.version}" "" ""', manpage) + manpage = manpage.replace('%(prog)s', parser.prog) # Correct quoting for single quotes. See groff manpage. manpage = manpage.replace('`', '\\(cq') diff -Nru osm2pgsql-1.8.1+ds/docs/osm2pgsql.1 osm2pgsql-1.9.0+ds/docs/osm2pgsql.1 --- osm2pgsql-1.8.1+ds/docs/osm2pgsql.1 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/docs/osm2pgsql.1 2023-08-15 19:18:18.000000000 +0000 @@ -1,7 +1,7 @@ -.TH "OSM2PGSQL" "1" "1.8.1" "" "" +.TH "OSM2PGSQL" "1" "1.9.0" "" "" .SH NAME .PP -osm2pgsql - Openstreetmap data to PostgreSQL converter +osm2pgsql - OpenStreetMap data to PostgreSQL converter .SH SYNOPSIS .PP \f[B]osm2pgsql\f[R] [\f[I]OPTIONS\f[R]] OSM-FILE\&... @@ -19,7 +19,7 @@ required by the configuration and import the OSM file(s) specified on the command line into those tables. Note that you also have to use the \f[B]-s, --slim\f[R] option if you -want your database to be updateable. +want your database to be updatable. .PP In \[lq]append\[rq] mode osm2pgsql will update the database tables with the data from OSM change files specified on the command line. @@ -93,6 +93,12 @@ .TP -P, --port=PORT Database server port. +.TP +--schema=SCHEMA +Default for various schema settings throughout osm2pgsql (default: +\f[V]public\f[R]). +The schema must exist in the database and be writable by the database +user. .SH INPUT OPTIONS .TP -r, --input-reader=FORMAT @@ -186,13 +192,33 @@ .TP --middle-schema=SCHEMA Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in -the middle (default is no schema, i.e.\ the \f[V]public\f[R] schema is -used). +the middle. +The schema must exist in the database and be writable by the database +user. +By default the schema set with \f[V]--schema\f[R] is used, or +\f[V]public\f[R] if that is not set. .TP --middle-way-node-index-id-shift=SHIFT Set ID shift for way node bucket index in middle. Experts only. See documentation for details. +.TP +--middle-database-format=FORMAT +Set the database format for the middle tables to FORMAT. +Allowed formats are \f[B]legacy\f[R] and \f[B]new\f[R]. +The \f[B]legacy\f[R] format is the old format that will eventually be +deprecated and removed but is currently still the default. +The \f[B]new\f[R] format was introduced in version 1.9.0 and is still +experimental. +See the manual for details on these formats. +(Only works with \f[B]--slim\f[R]. +In append mode osm2pgsql will automatically detect the database format, +so don\[cq]t use this with \f[B]-a, --append\f[R].) +.TP +--middle-with-nodes +Used together with the \f[B]new\f[R] middle database format when a flat +nodes file is used to force storing nodes with tags in the database, +too. .SH OUTPUT OPTIONS .TP -O, --output=OUTPUT @@ -292,8 +318,11 @@ .TP --output-pgsql-schema=SCHEMA Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in -the pgsql output (default is no schema, i.e.\ the \f[V]public\f[R] -schema is used). +the pgsql output. +The schema must exist in the database and be writable by the database +user. +By default the schema set with \f[V]--schema\f[R] is used, or +\f[V]public\f[R] if that is not set. .SH EXPIRE OPTIONS .TP -e, --expire-tiles=[MIN_ZOOM-]MAX-ZOOM diff -Nru osm2pgsql-1.8.1+ds/docs/osm2pgsql-gen.1 osm2pgsql-1.9.0+ds/docs/osm2pgsql-gen.1 --- osm2pgsql-1.8.1+ds/docs/osm2pgsql-gen.1 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/docs/osm2pgsql-gen.1 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,87 @@ +.TH "OSM2PGSQL" "1" "1.8.1" "" "" +.SH NAME +.PP +osm2pgsql-gen - Generalize OpenStreetMap data - EXPERIMENTAL! +.SH SYNOPSIS +.PP +\f[B]osm2pgsql-gen\f[R] [\f[I]OPTIONS\f[R]]\&... +.SH DESCRIPTION +.PP +THIS PROGRAM IS EXPERIMENTAL AND MIGHT CHANGE WITHOUT NOTICE! +.PP +\f[B]osm2pgsql-gen\f[R] reads data imported by \f[B]osm2pgsql\f[R] from +the database, performs various generalization steps specified by a Lua +config file and writes the data back to the database. +It is used in conjunction with and after \f[B]osm2pgsql\f[R] and reads +the same config file. +.PP +This man page can only cover some of the basics and describe the command +line options. +See the Generalization chapter in the osm2pgsql +Manual (https://osm2pgsql.org/doc/manual.html#generalization) for more +information. +.SH OPTIONS +.PP +This program follows the usual GNU command line syntax, with long +options starting with two dashes (\f[C]--\f[R]). +Mandatory arguments to long options are mandatory for short options too. +.SH MAIN OPTIONS +.TP +-a, --append +Run in append mode. +.TP +-c, --create +Run in create mode. +This is the default if \f[B]-a, --append\f[R] is not specified. +.TP +-S, --style=FILE +The Lua config file. +Same as for \f[B]osm2pgsql\f[R]. +.TP +-j, -jobs=NUM +Specifies the number of parallel threads used for certain operations. +Setting this to the number of available CPU cores is a reasonable +starting point. +.SH HELP/VERSION OPTIONS +.TP +-h, --help +Print help. +.TP +-V, --version +Print osm2pgsql version. +.SH LOGGING OPTIONS +.TP +--log-level=LEVEL +Set log level (`debug', `info' (default), `warn', or `error'). +.TP +--log-sql +Enable logging of SQL commands for debugging. +.SH DATABASE OPTIONS +.TP +-d, --database=NAME +The name of the PostgreSQL database to connect to. +If this parameter contains an \f[C]=\f[R] sign or starts with a valid +URI prefix (\f[C]postgresql://\f[R] or \f[C]postgres://\f[R]), it is +treated as a conninfo string. +See the PostgreSQL manual for details. +.TP +-U, --username=NAME +Postgresql user name. +.TP +-W, --password +Force password prompt. +.TP +-H, --host=HOSTNAME +Database server hostname or unix domain socket location. +.TP +-P, --port=PORT +Database server port. +.SH SEE ALSO +.IP \[bu] 2 +osm2pgsql website (https://osm2pgsql.org) +.IP \[bu] 2 +osm2pgsql manual (https://osm2pgsql.org/doc/manual.html) +.IP \[bu] 2 +\f[B]postgres\f[R](1) +.IP \[bu] 2 +\f[B]osm2pgsql\f[R](1) diff -Nru osm2pgsql-1.8.1+ds/docs/osm2pgsql-gen.md osm2pgsql-1.9.0+ds/docs/osm2pgsql-gen.md --- osm2pgsql-1.8.1+ds/docs/osm2pgsql-gen.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/docs/osm2pgsql-gen.md 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,99 @@ +# NAME + +osm2pgsql-gen - Generalize OpenStreetMap data - EXPERIMENTAL! + +# SYNOPSIS + +**osm2pgsql-gen** \[*OPTIONS*\]... + +# DESCRIPTION + +THIS PROGRAM IS EXPERIMENTAL AND MIGHT CHANGE WITHOUT NOTICE! + +**osm2pgsql-gen** reads data imported by **osm2pgsql** from the database, +performs various generalization steps specified by a Lua config file and +writes the data back to the database. It is used in conjunction with and +after **osm2pgsql** and reads the same config file. + +This man page can only cover some of the basics and describe the command line +options. See the [Generalization chapter in the osm2pgsql +Manual](https://osm2pgsql.org/doc/manual.html#generalization) for more +information. + +# OPTIONS + +This program follows the usual GNU command line syntax, with long options +starting with two dashes (`--`). Mandatory arguments to long options are +mandatory for short options too. + +# MAIN OPTIONS + +-a, \--append +: Run in append mode. + +-c, \--create +: Run in create mode. This is the default if **-a, \--append** is not + specified. + +-S, \--style=FILE +: The Lua config file. Same as for **osm2pgsql**. Usually not required + because it is read from the `osm2pgsql_properties` table. + +-j, \-jobs=NUM +: Specifies the number of parallel threads used for certain operations. + Setting this to the number of available CPU cores is a reasonable starting + point. + +# HELP/VERSION OPTIONS + +-h, \--help +: Print help. + +-V, \--version +: Print osm2pgsql version. + +# LOGGING OPTIONS + +\--log-level=LEVEL +: Set log level ('debug', 'info' (default), 'warn', or 'error'). + +\--log-sql +: Enable logging of SQL commands for debugging. + +# DATABASE OPTIONS + +-d, \--database=NAME +: The name of the PostgreSQL database to connect to. If this parameter + contains an `=` sign or starts with a valid URI prefix (`postgresql://` or + `postgres://`), it is treated as a conninfo string. See the PostgreSQL + manual for details. + +-U, \--username=NAME +: Postgresql user name. + +-W, \--password +: Force password prompt. + +-H, \--host=HOSTNAME +: Database server hostname or unix domain socket location. + +-P, \--port=PORT +: Database server port. + +\--schema=SCHEMA +: Default for various schema settings throughout osm2pgsql-gen + (default: `public`). The schema must exist in the database and be writable + by the database user. It must be the same as used with `osm2pgsql`. + +\--middle-schema=SCHEMA +: Database schema where the `osm2pgsql_properties` table is to be found. + Default set with `--schema`. Set to the same value as on the `osm2pgsql` + command line. + +# SEE ALSO + +* [osm2pgsql website](https://osm2pgsql.org) +* [osm2pgsql manual](https://osm2pgsql.org/doc/manual.html) +* **postgres**(1) +* **osm2pgsql**(1) + diff -Nru osm2pgsql-1.8.1+ds/docs/osm2pgsql.md osm2pgsql-1.9.0+ds/docs/osm2pgsql.md --- osm2pgsql-1.8.1+ds/docs/osm2pgsql.md 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/docs/osm2pgsql.md 2023-08-15 19:18:18.000000000 +0000 @@ -1,6 +1,6 @@ # NAME -osm2pgsql - Openstreetmap data to PostgreSQL converter +osm2pgsql - OpenStreetMap data to PostgreSQL converter # SYNOPSIS @@ -18,7 +18,7 @@ In "create" mode osm2pgsql will create the database tables required by the configuration and import the OSM file(s) specified on the command line into those tables. Note that you also have to use the **-s, \--slim** option if you -want your database to be updateable. +want your database to be updatable. In "append" mode osm2pgsql will update the database tables with the data from OSM change files specified on the command line. @@ -91,6 +91,10 @@ -P, \--port=PORT : Database server port. +\--schema=SCHEMA +: Default for various schema settings throughout osm2pgsql (default: `public`). + The schema must exist in the database and be writable by the database user. + # INPUT OPTIONS -r, \--input-reader=FORMAT @@ -164,13 +168,28 @@ as it doesn't work well with small imports. The default is disabled. \--middle-schema=SCHEMA -: Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in - the middle (default is no schema, i.e. the `public` schema is used). +: Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in the + middle. The schema must exist in the database and be writable by the + database user. By default the schema set with `--schema` is used, or + `public` if that is not set. \--middle-way-node-index-id-shift=SHIFT : Set ID shift for way node bucket index in middle. Experts only. See documentation for details. +\--middle-database-format=FORMAT +: Set the database format for the middle tables to FORMAT. Allowed formats + are **legacy** and **new**. The **legacy** format is the old format that + will eventually be deprecated and removed but is currently still the + default. The **new** format was introduced in version 1.9.0 and is still + experimental. See the manual for details on these formats. (Only works + with **\--slim**. In append mode osm2pgsql will automatically detect the + database format, so don't use this with **-a, \--append**.) + +\--middle-with-nodes +: Used together with the **new** middle database format when a flat nodes + file is used to force storing nodes with tags in the database, too. + # OUTPUT OPTIONS -O, \--output=OUTPUT @@ -261,9 +280,10 @@ different projection is used for the geometries. \--output-pgsql-schema=SCHEMA -: Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in - the pgsql output (default is no schema, i.e. the `public` schema - is used). +: Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in the + pgsql output. The schema must exist in the database and be writable by the + database user. By default the schema set with `--schema` is used, or + `public` if that is not set. # EXPIRE OPTIONS diff -Nru osm2pgsql-1.8.1+ds/docs/osm2pgsql-replication.1 osm2pgsql-1.9.0+ds/docs/osm2pgsql-replication.1 --- osm2pgsql-1.8.1+ds/docs/osm2pgsql-replication.1 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/docs/osm2pgsql-replication.1 2023-08-15 19:18:18.000000000 +0000 @@ -1,4 +1,4 @@ -.TH "OSM2PGSQL-REPLICATION" "1" "1.8.1" "" "" +.TH "OSM2PGSQL-REPLICATION" "1" "1.9.0" "" "" .SH NAME osm2pgsql-replication \- osm2pgsql database updater .SH SYNOPSIS @@ -39,36 +39,71 @@ .SH OPTIONS 'osm2pgsql-replication init' usage: osm2pgsql-replication init [-h] [-q] [-v] [-d DB] [-U NAME] [-H HOST] [-P PORT] [-p PREFIX] - [--middle-schema MIDDLE_SCHEMA] + [--middle-schema SCHEMA] [--schema SCHEMA] [--osm-file FILE | --server URL] + [--start-at TIME] Initialise the replication process. .br .br -There are two ways to initialise the replication process: if you have imported +This function sets the replication service to use and determines from .br -from a file that contains replication source information, then the +which date to apply updates. You must call this function at least once .br -initialisation process can use this and set up replication from there. +to set up the replication process. It can safely be called again later .br -Use the command \(cq%(prog)s \-\-osm\-file \(cq for this. +to change the replication servers or to roll back the update process and +.br +reapply updates. .br .br -If the file has no replication information or you don't have the initial +There are different methods available for initialisation. When no +.br +additional parameters are given, the data is initialised from the data +.br +in the database. If the data was imported from a file with replication +.br +information and the properties table is available (for osm2pgsql >= 1.9) .br -import file anymore then replication can be set up according to +then the replication from the file is used. Otherwise the minutely .br -the data found in the database. It checks the planet_osm_way table for the +update service from openstreetmap.org is used as the default replication .br -newest way in the database and then queries the OSM API when the way was +service. The start date is either taken from the database timestamp .br -created. The date is used as the start date for replication. In this mode +(for osm2pgsql >= 1.9) or determined from the newest way in the database .br -the minutely diffs from the OSM servers are used as a source. You can change +by querying the OSM API about its creation date. .br -this with the \(cq\-\-server\(cq parameter. + +.br +The replication service can be changed with the \(cq\-\-server\(cq parameter. +.br +To use a different start date, add \(cq\-\-start\-at\(cq with an absolute +.br +ISO timestamp (e.g. 2007\-08\-20T12:21:53Z). When the program determines the +.br +start date from the database timestamp or way creation date, then it +.br +subtracts another 3 hours by default to ensure that all new changes are +.br +available. To change this rollback period, use \(cq\-\-start\-at\(cq with the +.br +number of minutes to rollback. This rollback mode can also be used to +.br +force initialisation to use the database date and ignore the date +.br +from the replication information in the file. +.br + +.br +The initialisation process can also use replication information from +.br +an OSM file directly and ignore all other date information. +.br +Use the command \(cqosm2pgsql-replication \-\-osm\-file \(cq for this. @@ -101,16 +136,24 @@ Prefix for table names (default 'planet_osm') .TP -\fB\-\-middle\-schema\fR MIDDLE_SCHEMA +\fB\-\-middle\-schema\fR SCHEMA Name of the schema to store the table for the replication state in .TP +\fB\-\-schema\fR SCHEMA +Name of the schema for the database + +.TP \fB\-\-osm\-file\fR FILE Get replication information from the given file. .TP \fB\-\-server\fR URL -Use replication server at the given URL (default: https://planet.openstreetmap.org/replication/minute) +Use replication server at the given URL + +.TP +\fB\-\-start\-at\fR TIME +Time when to start replication. When an absolute timestamp (in ISO format) is given, it will be used. If a number is given, then replication starts the number of minutes before the known date of the database. .SH OPTIONS 'osm2pgsql-replication update' usage: osm2pgsql-replication update update [options] [-- param [param ...]] @@ -214,14 +257,19 @@ Prefix for table names (default 'planet_osm') .TP -\fB\-\-middle\-schema\fR MIDDLE_SCHEMA +\fB\-\-middle\-schema\fR SCHEMA Name of the schema to store the table for the replication state in +.TP +\fB\-\-schema\fR SCHEMA +Name of the schema for the database + .SH OPTIONS 'osm2pgsql-replication status' usage: osm2pgsql-replication status [-h] [-q] [-v] [-d DB] [-U NAME] [-H HOST] [-P PORT] [-p PREFIX] - [--middle-schema MIDDLE_SCHEMA] [--json] + [--middle-schema SCHEMA] [--schema SCHEMA] + [--json] Print information about the current replication status, optionally as JSON. .br @@ -333,9 +381,13 @@ Prefix for table names (default 'planet_osm') .TP -\fB\-\-middle\-schema\fR MIDDLE_SCHEMA +\fB\-\-middle\-schema\fR SCHEMA Name of the schema to store the table for the replication state in +.TP +\fB\-\-schema\fR SCHEMA +Name of the schema for the database + .SH SEE ALSO * osm2pgsql website (https://osm2pgsql.org) .br diff -Nru osm2pgsql-1.8.1+ds/flex-config/addresses.lua osm2pgsql-1.9.0+ds/flex-config/addresses.lua --- osm2pgsql-1.8.1+ds/flex-config/addresses.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/addresses.lua 2023-08-15 19:18:18.000000000 +0000 @@ -19,7 +19,7 @@ } }) -function get_address(tags) +local function get_address(tags) local addr = {} local count = 0 diff -Nru osm2pgsql-1.8.1+ds/flex-config/attributes.lua osm2pgsql-1.9.0+ds/flex-config/attributes.lua --- osm2pgsql-1.8.1+ds/flex-config/attributes.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/attributes.lua 2023-08-15 19:18:18.000000000 +0000 @@ -49,7 +49,7 @@ { column = 'members', type = 'jsonb' }, }) -function format_date(ts) +local function format_date(ts) return os.date('!%Y-%m-%dT%H:%M:%SZ', ts) end diff -Nru osm2pgsql-1.8.1+ds/flex-config/bbox.lua osm2pgsql-1.9.0+ds/flex-config/bbox.lua --- osm2pgsql-1.8.1+ds/flex-config/bbox.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/bbox.lua 2023-08-15 19:18:18.000000000 +0000 @@ -32,7 +32,7 @@ -- Helper function to remove some of the tags we usually are not interested in. -- Returns true if there are no tags left. -function clean_tags(tags) +local function clean_tags(tags) tags.odbl = nil tags.created_by = nil tags.source = nil @@ -43,7 +43,7 @@ -- Helper function that looks at the tags and decides if this is possibly -- an area. -function has_area_tags(tags) +local function has_area_tags(tags) if tags.area == 'yes' then return true end @@ -81,8 +81,8 @@ -- Format the bounding box we get from calling get_bbox() on the parameter -- in the way needed for the PostgreSQL/PostGIS box2d type. -function format_bbox(object) - xmin, ymin, xmax, ymax = object.get_bbox() +local function format_bbox(object) + local xmin, ymin, xmax, ymax = object.get_bbox() if xmin == nil then return nil end diff -Nru osm2pgsql-1.8.1+ds/flex-config/compatible.lua osm2pgsql-1.9.0+ds/flex-config/compatible.lua --- osm2pgsql-1.8.1+ds/flex-config/compatible.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/compatible.lua 2023-08-15 19:18:18.000000000 +0000 @@ -40,6 +40,11 @@ -- hstore column. local hstore_column = nil +-- If this is set, area calculations are always in Mercator coordinates units +-- irrespective of the srid setting. +-- Set this to true if you used --reproject-area before. +local reproject_area = false + -- There is some very old specialized handling of route relations in osm2pgsql, -- which you probably don't need. This is disabled here, but you can enable -- it by setting this to true. If you don't understand this, leave it alone. @@ -52,9 +57,8 @@ end -- Used for splitting up long linestrings -if srid == 4326 then - max_length = 1 -else +local max_length = 1 +if srid == 3857 then max_length = 100000 end @@ -320,8 +324,8 @@ 'wood', } -function gen_columns(text_columns, with_hstore, area, geometry_type) - columns = {} +local function gen_columns(text_columns, with_hstore, area, geometry_type) + local columns = {} local add_column = function (name, type) columns[#columns + 1] = { column = name, type = type } @@ -333,12 +337,8 @@ add_column('z_order', 'int') - if area ~= nil then - if area then - add_column('way_area', 'area') - else - add_column('way_area', 'real') - end + if area then + add_column('way_area', 'real') end if hstore_column then @@ -361,13 +361,13 @@ tables.point = osm2pgsql.define_table{ name = prefix .. '_point', ids = { type = 'node', id_column = 'osm_id' }, - columns = gen_columns(point_columns, hstore or hstore_all, nil, 'point') + columns = gen_columns(point_columns, hstore or hstore_all, false, 'point') } tables.line = osm2pgsql.define_table{ name = prefix .. '_line', ids = { type = 'way', id_column = 'osm_id' }, - columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring') + columns = gen_columns(non_point_columns, hstore or hstore_all, true, 'linestring') } tables.polygon = osm2pgsql.define_table{ @@ -379,7 +379,7 @@ tables.roads = osm2pgsql.define_table{ name = prefix .. '_roads', ids = { type = 'way', id_column = 'osm_id' }, - columns = gen_columns(non_point_columns, hstore or hstore_all, false, 'linestring') + columns = gen_columns(non_point_columns, hstore or hstore_all, true, 'linestring') } local z_order_lookup = { @@ -412,11 +412,11 @@ motorway = {39, true} } -function as_bool(value) +local function as_bool(value) return value == 'yes' or value == 'true' or value == '1' end -function get_z_order(tags) +local function get_z_order(tags) local z_order = 100 * math.floor(tonumber(tags.layer or '0') or 0) local roads = false @@ -447,7 +447,7 @@ return z_order, roads end -function make_check_in_list_func(list) +local function make_check_in_list_func(list) local h = {} for _, k in ipairs(list) do h[k] = true @@ -465,7 +465,7 @@ local is_polygon = make_check_in_list_func(polygon_keys) local clean_tags = osm2pgsql.make_clean_tags_func(delete_keys) -function make_column_hash(columns) +local function make_column_hash(columns) local h = {} for _, k in ipairs(columns) do @@ -475,7 +475,7 @@ return h end -function make_get_output(columns, hstore_all) +local function make_get_output(columns) local h = make_column_hash(columns) if hstore_all then return function(tags) @@ -511,10 +511,10 @@ local has_generic_tag = make_check_in_list_func(generic_keys) -local get_point_output = make_get_output(point_columns, hstore_all) -local get_non_point_output = make_get_output(non_point_columns, hstore_all) +local get_point_output = make_get_output(point_columns) +local get_non_point_output = make_get_output(non_point_columns) -function get_hstore_column(tags) +local function get_hstore_column(tags) local len = #hstore_column local h = {} for k, v in pairs(tags) do @@ -561,7 +561,7 @@ tables.point:insert(output) end -function add_line(output, geom, roads) +local function add_line(output, geom, roads) for sgeom in geom:segmentize(max_length):geometries() do output.way = sgeom tables.line:insert(output) @@ -571,6 +571,20 @@ end end +local function compute_geom_and_area(geom) + local area, projected_geom + + projected_geom = geom:transform(srid) + + if reproject_area and srid ~= 3857 then + area = geom:transform(3857):area() + else + area = projected_geom:area() + end + + return projected_geom, area +end + function osm2pgsql.process_way(object) if clean_tags(object.tags) then return @@ -629,7 +643,9 @@ end if polygon and object.is_closed then - output.way = object:as_polygon() + local pgeom, area = compute_geom_and_area(object:as_polygon()) + output.way = pgeom + output.way_area = area tables.polygon:insert(output) else add_line(output, object:as_linestring(), roads) @@ -731,12 +747,17 @@ if make_boundary or make_polygon then local geom = object:as_multipolygon() + if multi_geometry then - output.way = geom + local pgeom, area = compute_geom_and_area(geom) + output.way = pgeom + output.way_area = area tables.polygon:insert(output) else for sgeom in geom:geometries() do - output.way = sgeom + local pgeom, area = compute_geom_and_area(sgeom) + output.way = pgeom + output.way_area = area tables.polygon:insert(output) end end diff -Nru osm2pgsql-1.8.1+ds/flex-config/data-types.lua osm2pgsql-1.9.0+ds/flex-config/data-types.lua --- osm2pgsql-1.8.1+ds/flex-config/data-types.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/data-types.lua 2023-08-15 19:18:18.000000000 +0000 @@ -30,7 +30,7 @@ -- Helper function to remove some of the tags we usually are not interested in. -- Returns true if there are no tags left. -function clean_tags(tags) +local function clean_tags(tags) tags.odbl = nil tags.created_by = nil tags.source = nil @@ -63,7 +63,7 @@ end -- Parse a maxspeed value like "30" or "55 mph" and return a number in km/h -function parse_speed(input) +local function parse_speed(input) if not input then return nil end diff -Nru osm2pgsql-1.8.1+ds/flex-config/expire.lua osm2pgsql-1.9.0+ds/flex-config/expire.lua --- osm2pgsql-1.8.1+ds/flex-config/expire.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/expire.lua 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,165 @@ +-- This config example file is released into the Public Domain. +-- +-- This examples shows how to use tile expiry. + +local expire_outputs = {} + +expire_outputs.pois = osm2pgsql.define_expire_output({ + -- The zoom level at which we calculate the tiles. This must always be set. + maxzoom = 14, + -- The filename where tile list should be written to. + filename = 'pois.tiles' +}) + +expire_outputs.lines = osm2pgsql.define_expire_output({ + maxzoom = 14, + -- Instead of writing the tile list to a file, it can be written to a table. + -- The table will be created if it isn't there already. + table = 'lines_tiles', +-- schema = 'myschema', -- You can also set a database schema. +}) + +expire_outputs.polygons = osm2pgsql.define_expire_output({ + -- You can also set a minimum zoom level in addition to the maximum zoom + -- level. Tiles in all zoom levels between those two will be written out. + minzoom = 10, + maxzoom = 14, + table = 'polygons_tiles' +}) + +print("Expire outputs:(") +for name, eo in pairs(expire_outputs) do + print(" " .. name + .. ": minzoom=" .. eo:minzoom() + .. " maxzoom=" .. eo:maxzoom() + .. " filename=" .. eo:filename() + .. " schema=" .. eo:schema() + .. " table=" .. eo:table()) +end +print(")") + +local tables = {} + +tables.pois = osm2pgsql.define_node_table('pois', { + { column = 'tags', type = 'jsonb' }, + -- Zero, one or more expire outputs are referenced in an `expire` field in + -- the definition of any geometry column using the Web Mercator (3857) + -- projection. + { column = 'geom', type = 'point', not_null = true, expire = { { output = expire_outputs.pois } } }, +}) + +tables.lines = osm2pgsql.define_way_table('lines', { + { column = 'tags', type = 'jsonb' }, + -- If you only have a single expire output and with the default + -- parameters, you can specify it directly. + { column = 'geom', type = 'linestring', not_null = true, expire = expire_outputs.lines }, +}) + +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'tags', type = 'jsonb' }, + -- In this configuration the `mode` of the expiry is set to `boundary-only`. + -- Other modes are `full-area` (default) and `hybrid`. If set to `hybrid` + -- you can set `full_area_limit` to a value in Web Mercator units. For + -- polygons where the width and height of the bounding box is below this + -- limit the full area is expired, for larger polygons only the boundary. + -- This setting doesn't have any effect on point or linestring geometries. + { column = 'geom', type = 'geometry', not_null = true, expire = { + { output = expire_outputs.polygons, mode = 'boundary-only' } + }} +}) + +tables.boundaries = osm2pgsql.define_relation_table('boundaries', { + { column = 'type', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + -- This geometry column doesn't have an `expire` field, so no expiry is + -- done. + { column = 'geom', type = 'multilinestring', not_null = true }, +}) + +print("Tables:(") +for name, ts in pairs(tables) do + print(" " .. name .. ": name=" .. ts:name() .. " (" .. tostring(ts) .. ")") +end +print(")") + +-- Helper function that looks at the tags and decides if this is possibly +-- an area. +local function has_area_tags(tags) + if tags.area == 'yes' then + return true + end + if tags.area == 'no' then + return false + end + + return tags.aeroway + or tags.amenity + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] +end + +function osm2pgsql.process_node(object) + tables.pois:insert({ + tags = object.tags, + geom = object:as_point() + }) +end + +function osm2pgsql.process_way(object) + if object.is_closed and has_area_tags(object.tags) then + tables.polygons:insert({ + tags = object.tags, + geom = object:as_polygon() + }) + else + tables.lines:insert({ + tags = object.tags, + geom = object:as_linestring() + }) + end +end + +function osm2pgsql.process_relation(object) + local relation_type = object:grab_tag('type') + + -- Store boundary relations as multilinestrings + if relation_type == 'boundary' then + tables.boundaries:insert({ + type = object:grab_tag('boundary'), + tags = object.tags, + geom = object:as_multilinestring():line_merge() + }) + return + end + + -- Store multipolygon relations as polygons + if relation_type == 'multipolygon' then + tables.polygons:insert({ + tags = object.tags, + geom = object:as_multipolygon() + }) + end +end + diff -Nru osm2pgsql-1.8.1+ds/flex-config/gen/forests.lua osm2pgsql-1.9.0+ds/flex-config/gen/forests.lua --- osm2pgsql-1.8.1+ds/flex-config/gen/forests.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/gen/forests.lua 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,122 @@ +-- This config example file is released into the Public Domain. + +-- This Lua config demonstrates how to use (some of) the generalization +-- functionality. In this case we are generating a forest layer and create +-- several levels of generalized data from it. +-- +-- NOTE THAT THE GENERALIZATION SUPPORT IS EXPERIMENTAL AND MIGHT CHANGE +-- WITHOUT NOTICE! + +-- We want to do three levels of generalized data for different zoom levels +-- or scales: small, medium, and large. +local gen_levels = { 's', 'm', 'l' } + +-- And these are the zoom levels we do the generalization in. This doesn't +-- mean that these are the zoom levels for which the generalization is shown! +local zoom_levels = { s = 7, m = 8, l = 10 } + +local expire_outputs = {} + +-- These defines the expire outputs for three levels of generalized data. +-- See flex-config/expire.lua for details on how this works. +for _, level in ipairs(gen_levels) do + expire_outputs['exp_' .. level] = osm2pgsql.define_expire_output({ + maxzoom = zoom_levels[level], + table = 'exp_' .. level, + }) +end + +local tables = {} + +-- This is the table for the original data. +tables.forests = osm2pgsql.define_area_table('forests', { + -- We'll keep the name tag + { column = 'name', type = 'text' }, + -- We'll also keep all other tags, just because we can + { column = 'tags', type = 'jsonb' }, + -- Geometries can be polygons or multipolygons + { column = 'geom', type = 'geometry', not_null = true, + expire = { + { output = expire_outputs.exp_s }, + { output = expire_outputs.exp_m }, + { output = expire_outputs.exp_l }, + } + }, + -- If the forest has a name and isn't too small, this will be a point + -- where we can put a label. + { column = 'labelpoint', type = 'point' }, + -- The area (in Web Mercator units) for lager forests, otherwise NULL. + { column = 'area', type = 'real' }, +}) + +-- These defines the tables for three levels of generalized data. In this +-- case, the geometries generated by the generalization are always polygons. +for _, level in ipairs(gen_levels) do + tables['forests_' .. level] = osm2pgsql.define_table({ + name = 'forests_' .. level, + -- Define the x and y integer columns and add an index for them. + ids = { type = 'tile' }, + columns = { + { column = 'geom', type = 'polygon', not_null = true } + } + }) +end + +-- This is where the data is actually inserted into the forests table. If +-- the forest has a name tag and the area is larger than a defined minimum +-- we'll also add the name and a point for the label. +local minimum_area_for_label = 0.001 +local function insert_forest(tags, geom) + local attrs = { + tags = tags, + geom = geom, + } + if tags.name then + local area = geom:area() + if area >= minimum_area_for_label then + attrs.name = tags.name + attrs.area = area + attrs.labelpoint = geom:pole_of_inaccessibility() + end + end + tables.forests:insert(attrs) +end + +function osm2pgsql.process_way(object) + if not object.is_closed then + return + end + local tags = object.tags + if tags.natural == 'wood' or tags.landuse == 'forest' then + insert_forest(tags, object:as_polygon()) + end +end + +function osm2pgsql.process_relation(object) + if object.tags.type ~= 'multipolygon' then + return + end + local tags = object.tags + if tags.natural == 'wood' or tags.landuse == 'forest' then + insert_forest(tags, object:as_multipolygon()) + end +end + +-- This function is called in the generalization step. We define three levels +-- of generalization that should go into the already created tables. +function osm2pgsql.process_gen() + for _, level in ipairs(gen_levels) do + osm2pgsql.run_gen('raster-union', { + name = 'forests_' .. level, -- name (for logging) + debug = false, -- set to true to get more details debug output + src_table = 'forests', + dest_table = 'forests_' .. level, + zoom = zoom_levels[level], + geom_column = 'geom', + margin = 0.1, -- 10% overlap at tile boundaries + expire_list = 'exp_' .. level, -- where to get expired tiles in append mode + make_valid = true -- make sure geometries are valid + }) + end +end + diff -Nru osm2pgsql-1.8.1+ds/flex-config/gen/README.md osm2pgsql-1.9.0+ds/flex-config/gen/README.md --- osm2pgsql-1.8.1+ds/flex-config/gen/README.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/gen/README.md 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,16 @@ +# Flex Output Configuration for Generalization + +These are config file examples for use with the generalization support. + +GENERALIZATION SUPPORT IS EXPERIMENTAL. EVERYTHING IN THIS DIRECTORY MIGHT +CHANGE WITHOUT NOTICE. + +See the [Flex Output](https://osm2pgsql.org/doc/manual.html#the-flex-output) +and [Generalization](https://osm2pgsql.org/doc/manual.html#generalization) +chapters in the manual for all the details. + +## Public Domain + +All the example config files in this directory are released into the Public +Domain. You may use them in any way you like. + diff -Nru osm2pgsql-1.8.1+ds/flex-config/generic.lua osm2pgsql-1.9.0+ds/flex-config/generic.lua --- osm2pgsql-1.8.1+ds/flex-config/generic.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/generic.lua 2023-08-15 19:18:18.000000000 +0000 @@ -22,7 +22,6 @@ tables.polygons = osm2pgsql.define_area_table('polygons', { { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'geometry', projection = srid, not_null = true }, - { column = 'area', type = 'area' }, }) tables.routes = osm2pgsql.define_relation_table('routes', { @@ -183,7 +182,7 @@ -- Helper function that looks at the tags and decides if this is possibly -- an area. -function has_area_tags(tags) +local function has_area_tags(tags) if tags.area == 'yes' then return true end diff -Nru osm2pgsql-1.8.1+ds/flex-config/geometries.lua osm2pgsql-1.9.0+ds/flex-config/geometries.lua --- osm2pgsql-1.8.1+ds/flex-config/geometries.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/geometries.lua 2023-08-15 19:18:18.000000000 +0000 @@ -34,7 +34,10 @@ -- if we have multiple geometry columns and some of them can be valid -- and others not. { column = 'geom', type = 'geometry', projection = 4326 }, + -- In this column we'll put the area calculated in Mercator coordinates { column = 'area', type = 'real' }, + -- In this column we'll put the true area calculated on the spheroid + { column = 'spherical_area', type = 'real' }, }) tables.boundaries = osm2pgsql.define_relation_table('boundaries', { @@ -54,7 +57,7 @@ -- Helper function to remove some of the tags we usually are not interested in. -- Returns true if there are no tags left. -function clean_tags(tags) +local function clean_tags(tags) tags.odbl = nil tags.created_by = nil tags.source = nil @@ -65,7 +68,7 @@ -- Helper function that looks at the tags and decides if this is possibly -- an area. -function has_area_tags(tags) +local function has_area_tags(tags) if tags.area == 'yes' then return true end @@ -135,7 +138,9 @@ geom = geom, -- Calculate the area in Mercator projection and store in the -- area column - area = geom:transform(3857):area() + area = geom:transform(3857):area(), + -- Also calculate "real" area in spheroid + spherical_area = geom:spherical_area() }) else -- We want to split long lines into smaller segments. We can use @@ -192,7 +197,9 @@ geom = geom, -- Calculate the area in Mercator projection and store in the -- area column - area = geom:transform(3857):area() + area = geom:transform(3857):area(), + -- Also calculate "real" area in spheroid + spherical_area = geom:spherical_area() }) end end diff -Nru osm2pgsql-1.8.1+ds/flex-config/indexes.lua osm2pgsql-1.9.0+ds/flex-config/indexes.lua --- osm2pgsql-1.8.1+ds/flex-config/indexes.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/indexes.lua 2023-08-15 19:18:18.000000000 +0000 @@ -71,7 +71,7 @@ -- Helper function that looks at the tags and decides if this is possibly -- an area. -function has_area_tags(tags) +local function has_area_tags(tags) if tags.area == 'yes' then return true end diff -Nru osm2pgsql-1.8.1+ds/flex-config/labelpoint.lua osm2pgsql-1.9.0+ds/flex-config/labelpoint.lua --- osm2pgsql-1.8.1+ds/flex-config/labelpoint.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/labelpoint.lua 2023-08-15 19:18:18.000000000 +0000 @@ -15,7 +15,7 @@ { column = 'poi2', type = 'point', not_null = true }, }) -function add(tags, geom) +local function add(tags, geom) tables.polygons:insert({ name = tags.name, tags = tags, diff -Nru osm2pgsql-1.8.1+ds/flex-config/route-relations.lua osm2pgsql-1.9.0+ds/flex-config/route-relations.lua --- osm2pgsql-1.8.1+ds/flex-config/route-relations.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/route-relations.lua 2023-08-15 19:18:18.000000000 +0000 @@ -31,7 +31,7 @@ -- it can be called any number of times and will lead to the same result. local w2r = {} -function clean_tags(tags) +local function clean_tags(tags) tags.odbl = nil tags.created_by = nil tags.source = nil diff -Nru osm2pgsql-1.8.1+ds/flex-config/simple.lua osm2pgsql-1.9.0+ds/flex-config/simple.lua --- osm2pgsql-1.8.1+ds/flex-config/simple.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/simple.lua 2023-08-15 19:18:18.000000000 +0000 @@ -71,7 +71,7 @@ -- Helper function to remove some of the tags we usually are not interested in. -- Returns true if there are no tags left. -function clean_tags(tags) +local function clean_tags(tags) tags.odbl = nil tags.created_by = nil tags.source = nil diff -Nru osm2pgsql-1.8.1+ds/flex-config/unitable.lua osm2pgsql-1.9.0+ds/flex-config/unitable.lua --- osm2pgsql-1.8.1+ds/flex-config/unitable.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/flex-config/unitable.lua 2023-08-15 19:18:18.000000000 +0000 @@ -25,7 +25,7 @@ -- Helper function to remove some of the tags we usually are not interested in. -- Returns true if there are no tags left. -function clean_tags(tags) +local function clean_tags(tags) tags.odbl = nil tags.created_by = nil tags.source = nil @@ -34,7 +34,7 @@ return next(tags) == nil end -function process(object, geometry) +local function process(object, geometry) if clean_tags(object.tags) then return end diff -Nru osm2pgsql-1.8.1+ds/.github/actions/build-and-test/action.yml osm2pgsql-1.9.0+ds/.github/actions/build-and-test/action.yml --- osm2pgsql-1.8.1+ds/.github/actions/build-and-test/action.yml 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.github/actions/build-and-test/action.yml 2023-08-15 19:18:18.000000000 +0000 @@ -1,4 +1,4 @@ -name: Build and test osm2pgsq +name: Build and test osm2pgsql inputs: test-wrapper: diff -Nru osm2pgsql-1.8.1+ds/.github/actions/ubuntu-prerequisites/action.yml osm2pgsql-1.9.0+ds/.github/actions/ubuntu-prerequisites/action.yml --- osm2pgsql-1.8.1+ds/.github/actions/ubuntu-prerequisites/action.yml 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.github/actions/ubuntu-prerequisites/action.yml 2023-08-15 19:18:18.000000000 +0000 @@ -16,12 +16,15 @@ - name: Install software run: | sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + cimg-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ libexpat1-dev \ + libpotrace-dev \ libpq-dev \ libproj-dev \ + nlohmann-json3-dev \ pandoc \ postgresql-${POSTGRESQL_VERSION} \ postgresql-${POSTGRESQL_VERSION}-postgis-${POSTGIS_VERSION} \ @@ -30,7 +33,7 @@ python3-psycopg2 \ python3-setuptools \ zlib1g-dev - pip3 install behave + pip3 install behave osmium if [ "$CC" = clang-8 ]; then sudo apt-get install -yq --no-install-suggests --no-install-recommends clang-8; fi shell: bash diff -Nru osm2pgsql-1.8.1+ds/.github/actions/win-install/action.yml osm2pgsql-1.9.0+ds/.github/actions/win-install/action.yml --- osm2pgsql-1.8.1+ds/.github/actions/win-install/action.yml 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.github/actions/win-install/action.yml 2023-08-15 19:18:18.000000000 +0000 @@ -5,9 +5,21 @@ steps: - name: Install packages - run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows + run: | + vcpkg install \ + boost-filesystem:x64-windows \ + boost-geometry:x64-windows \ + boost-property-tree:x64-windows \ + boost-system:x64-windows \ + bzip2:x64-windows \ + cimg:x64-windows \ + expat:x64-windows \ + libpq:x64-windows \ + lua:x64-windows \ + nlohmann-json:x64-windows \ + proj4:x64-windows \ + zlib:x64-windows shell: bash - - - name: Install psycopg2 and beahve - run: python -m pip install psycopg2 behave + - name: Install psycopg2 and behave + run: python -m pip install psycopg2 behave osmium shell: bash diff -Nru osm2pgsql-1.8.1+ds/.github/workflows/ci.yml osm2pgsql-1.9.0+ds/.github/workflows/ci.yml --- osm2pgsql-1.8.1+ds/.github/workflows/ci.yml 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.github/workflows/ci.yml 2023-08-15 19:18:18.000000000 +0000 @@ -4,7 +4,14 @@ jobs: macos: - runs-on: macos-latest + strategy: + fail-fast: false + matrix: + os: + - "macos-11" + - "macos-12" # latest + - "macos-13" + runs-on: ${{ matrix.os }} env: LUA_VERSION: 5.4 @@ -14,8 +21,8 @@ - name: Install prerequisites run: | - brew install lua boost postgis pandoc - pip3 install psycopg2 behave + brew install lua boost postgis pandoc cimg potrace nlohmann-json + pip3 install psycopg2 behave osmium pg_ctl -D /usr/local/var/postgres init pg_ctl -D /usr/local/var/postgres start # Work around homebrew postgresql installation screw-up, see @@ -39,16 +46,17 @@ env: PGHOST: /tmp - ubuntu18-pg96-gcc7-jit: - runs-on: ubuntu-18.04 + ubuntu20-pg96-gcc10-jit: + runs-on: ubuntu-20.04 env: - CC: gcc-7 - CXX: g++-7 + CC: gcc-10 + CXX: g++-10 + EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug LUA_VERSION: 5.3 LUAJIT_OPTION: ON POSTGRESQL_VERSION: 9.6 - POSTGIS_VERSION: 2.4 + POSTGIS_VERSION: 2.5 BUILD_TYPE: Release steps: @@ -56,16 +64,16 @@ - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu18-pg96-clang8-jit: - runs-on: ubuntu-18.04 + ubuntu20-pg96-clang10-jit: + runs-on: ubuntu-20.04 env: - CC: clang-8 - CXX: clang++-8 + CC: clang-10 + CXX: clang++-10 LUA_VERSION: 5.3 LUAJIT_OPTION: ON POSTGRESQL_VERSION: 9.6 - POSTGIS_VERSION: 2.4 + POSTGIS_VERSION: 2.5 BUILD_TYPE: Release steps: @@ -73,17 +81,17 @@ - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test - ubuntu18-pg10-gcc9: - runs-on: ubuntu-18.04 + ubuntu20-pg10-gcc10: + runs-on: ubuntu-20.04 env: - CC: gcc-9 - CXX: g++-9 + CC: gcc-10 + CXX: g++-10 + EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug LUA_VERSION: 5.3 LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 10 POSTGIS_VERSION: 3 - CPP_VERSION: 17 BUILD_TYPE: Debug steps: @@ -92,17 +100,16 @@ - uses: ./.github/actions/build-and-test - ubuntu18-pg11-clang9: - runs-on: ubuntu-18.04 + ubuntu20-pg11-clang10: + runs-on: ubuntu-20.04 env: - CC: clang-9 - CXX: clang++-9 + CC: clang-10 + CXX: clang++-10 LUA_VERSION: 5.3 LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 11 POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 BUILD_TYPE: Debug steps: @@ -120,7 +127,6 @@ LUAJIT_OPTION: ON POSTGRESQL_VERSION: 12 POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 BUILD_TYPE: Debug steps: @@ -139,7 +145,6 @@ LUAJIT_OPTION: ON POSTGRESQL_VERSION: 13 POSTGIS_VERSION: 3 - CPP_VERSION: 17 BUILD_TYPE: Debug steps: @@ -157,7 +162,6 @@ LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 13 POSTGIS_VERSION: 3 - CPP_VERSION: 17 USE_PROJ_LIB: 6 BUILD_TYPE: Debug @@ -176,7 +180,6 @@ LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 13 POSTGIS_VERSION: 3 - CPP_VERSION: 17 USE_PROJ_LIB: off BUILD_TYPE: Debug @@ -195,7 +198,6 @@ LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 - CPP_VERSION: 17 USE_PROJ_LIB: 6 BUILD_TYPE: Debug @@ -214,7 +216,6 @@ LUAJIT_OPTION: ON POSTGRESQL_VERSION: 13 POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 BUILD_TYPE: Release steps: @@ -230,7 +231,6 @@ CXX: g++-10 POSTGRESQL_VERSION: 13 POSTGIS_VERSION: 2.5 - CPP_VERSION: 17 BUILD_TYPE: Release steps: @@ -248,7 +248,6 @@ LUAJIT_OPTION: ON POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 - CPP_VERSION: 17 BUILD_TYPE: Debug steps: @@ -266,7 +265,6 @@ LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 - CPP_VERSION: 17 USE_PROJ_LIB: 6 BUILD_TYPE: Debug @@ -285,7 +283,6 @@ LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 - CPP_VERSION: 17 USE_PROJ_LIB: off BUILD_TYPE: Debug @@ -304,7 +301,6 @@ LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 - CPP_VERSION: 17 USE_PROJ_LIB: 6 BUILD_TYPE: Debug @@ -324,7 +320,6 @@ LUAJIT_OPTION: ON POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 - CPP_VERSION: 17 BUILD_TYPE: Release steps: @@ -341,7 +336,6 @@ EXTRA_FLAGS: -Wno-stringop-overread POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 - CPP_VERSION: 17 BUILD_TYPE: Release steps: @@ -349,6 +343,42 @@ - uses: ./.github/actions/ubuntu-prerequisites - uses: ./.github/actions/build-and-test + ubuntu22-pg14-clang14-cpp20: + runs-on: ubuntu-22.04 + + env: + CC: clang-14 + CXX: clang++-14 + LUA_VERSION: 5.3 + LUAJIT_OPTION: OFF + POSTGRESQL_VERSION: 14 + POSTGIS_VERSION: 3 + CPP_VERSION: 20 + BUILD_TYPE: Debug + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/ubuntu-prerequisites + - uses: ./.github/actions/build-and-test + + ubuntu22-pg14-gcc12-cpp20: + runs-on: ubuntu-22.04 + + env: + CC: gcc-12 + CXX: g++-12 + LUA_VERSION: 5.3 + LUAJIT_OPTION: OFF + POSTGRESQL_VERSION: 14 + POSTGIS_VERSION: 3 + CPP_VERSION: 20 + BUILD_TYPE: Debug + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/ubuntu-prerequisites + - uses: ./.github/actions/build-and-test + windows: strategy: fail-fast: false @@ -367,7 +397,9 @@ with: path: | C:/vcpkg_binary_cache - key: vcpkg-binary-cache-${{ matrix.os }} + key: vcpkg-binary-cache-${{ matrix.os }}-${{ github.run_id }} + restore-keys: | + vcpkg-binary-cache-${{ matrix.os }} - uses: actions/cache@v3 with: path: | diff -Nru osm2pgsql-1.8.1+ds/.github/workflows/luacheck.yml osm2pgsql-1.9.0+ds/.github/workflows/luacheck.yml --- osm2pgsql-1.8.1+ds/.github/workflows/luacheck.yml 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.github/workflows/luacheck.yml 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,19 @@ +name: Check Lua scripts with luacheck + +on: [ push, pull_request ] + +jobs: + luacheck: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Install prerequisites + run: | + sudo apt-get update -qq + sudo apt-get install -yq --no-install-suggests --no-install-recommends lua-check + + - name: Run luacheck + run: luacheck flex-config/*.lua flex-config/gen/*.lua tests/data/*.lua tests/lua/tests.lua + diff -Nru osm2pgsql-1.8.1+ds/.github/workflows/test-install.yml osm2pgsql-1.9.0+ds/.github/workflows/test-install.yml --- osm2pgsql-1.8.1+ds/.github/workflows/test-install.yml 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.github/workflows/test-install.yml 2023-08-15 19:18:18.000000000 +0000 @@ -4,7 +4,7 @@ jobs: ubuntu-test-install: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: @@ -19,10 +19,10 @@ env: LUA_VERSION: 5.3 - POSTGRESQL_VERSION: 12 + POSTGRESQL_VERSION: 14 POSTGIS_VERSION: 3 BUILD_TYPE: Release - CXXFLAGS: -pedantic -Wextra -Werror + CXXFLAGS: -pedantic -Wextra -Wno-stringop-overread -Werror PREFIX: /usr/local OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf OSMFILE: monaco-latest.osm.pbf @@ -36,15 +36,19 @@ - name: Install prerequisites run: | sudo apt-get purge -yq postgresql* + sudo apt-get update -qq sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + cimg-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ libexpat1-dev \ liblua${LUA_VERSION}-dev \ libluajit-5.1-dev \ + libpotrace-dev \ libpq-dev \ libproj-dev \ + nlohmann-json3-dev \ lua${LUA_VERSION} \ pandoc \ postgresql-${POSTGRESQL_VERSION} \ diff -Nru osm2pgsql-1.8.1+ds/.luacheckrc osm2pgsql-1.9.0+ds/.luacheckrc --- osm2pgsql-1.8.1+ds/.luacheckrc 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/.luacheckrc 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,38 @@ +-- +-- Configuration for luacheck +-- +-- To check Lua files call like this: +-- luacheck flex-config/*.lua flex-config/gen/*.lua tests/data/*.lua tests/lua/tests.lua +-- + +unused_args = false + +stds.osm2pgsql = { + read_globals = { + osm2pgsql = { + fields = { + process_node = { + read_only = false + }, + process_way = { + read_only = false + }, + process_relation = { + read_only = false + }, + select_relation_members = { + read_only = false + }, + process_gen = { + read_only = false + }, + }, + other_fields = true, + } + } +} + +std = 'min+osm2pgsql' + +files['tests/lua/tests.lua'].globals = { 'osm2pgsql' } + diff -Nru osm2pgsql-1.8.1+ds/README.md osm2pgsql-1.9.0+ds/README.md --- osm2pgsql-1.8.1+ds/README.md 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/README.md 2023-08-15 19:18:18.000000000 +0000 @@ -49,6 +49,9 @@ * [zlib](https://www.zlib.net/) * [Boost libraries](https://www.boost.org/), including geometry, system and filesystem +* [nlohmann/json](https://json.nlohmann.me/) +* [CImg](https://cimg.eu/) (Optional, for generalization only) +* [potrace](https://potrace.sourceforge.net/) (Optional, for generalization only) * [PostgreSQL](https://www.postgresql.org/) client libraries * [Lua](https://www.lua.org/) (Optional, used for Lua tag transforms and the flex output) @@ -80,14 +83,16 @@ ```sh sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \ - libboost-filesystem-dev libexpat1-dev zlib1g-dev \ - libbz2-dev libpq-dev libproj-dev lua5.3 liblua5.3-dev pandoc + libboost-filesystem-dev libexpat1-dev zlib1g-dev libpotrace-dev cimg-dev \ + libbz2-dev libpq-dev libproj-dev lua5.3 liblua5.3-dev pandoc \ + nlohmann-json3-dev pyosmium ``` On a Fedora system, use ```sh sudo dnf install cmake make gcc-c++ boost-devel expat-devel zlib-devel \ + potrace-devel cimg-devel json-devel python3-osmium \ bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel pandoc ``` @@ -96,6 +101,7 @@ ```sh sudo yum install cmake make gcc-c++ boost-devel expat-devel zlib-devel \ + potrace-devel cimg-devel json-devel python3-osmium \ bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel pandoc ``` @@ -143,6 +149,12 @@ sudo make install ``` +To install the experimental `osm2pgsql-gen` binary use + +```sh +sudo make install-gen +``` + By default, the Release build with debug info is created and no tests are compiled. You can change that behavior by using additional options like following: @@ -199,6 +211,11 @@ Lua 5.1.4 (LuaJIT 2.1.0-beta3) ``` +## Generalization + +There is some experimental support for data generalization. See +https://osm2pgsql.org/generalization/ for details. + ## Help/Support If you have problems with osm2pgsql or want to report a bug, go to diff -Nru osm2pgsql-1.8.1+ds/scripts/osm2pgsql-replication osm2pgsql-1.9.0+ds/scripts/osm2pgsql-replication --- osm2pgsql-1.8.1+ds/scripts/osm2pgsql-replication 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/scripts/osm2pgsql-replication 2023-08-15 19:18:18.000000000 +0000 @@ -54,118 +54,283 @@ OSM2PGSQL_PATH = Path(__file__).parent.resolve() / 'osm2pgsql' def pretty_format_timedelta(seconds): - minutes = int(seconds/60) + seconds = int(seconds) + (minutes, seconds) = divmod(seconds, 60) (hours, minutes) = divmod(minutes, 60) (days, hours) = divmod(hours, 24) (weeks, days) = divmod(days, 7) - if 0 < seconds < 60: - output = "<1 minute" - else: - output = [] - # If weeks > 1 but hours == 0, we still want to show "0 hours" - if weeks > 0: - output.append("{} week(s)".format(weeks)) - if days > 0 or weeks > 0: - output.append("{} day(s)".format(days)) - if hours > 0 or days > 0 or weeks > 0: - output.append("{} hour(s)".format(hours)) - + output = [] + # If weeks > 1 but hours == 0, we still want to show "0 hours" + if weeks > 0: + output.append("{} week(s)".format(weeks)) + if days > 0 or weeks > 0: + output.append("{} day(s)".format(days)) + if hours > 0 or days > 0 or weeks > 0: + output.append("{} hour(s)".format(hours)) + if minutes > 0 or hours > 0 or days > 0 or weeks > 0: output.append("{} minute(s)".format(minutes)) - output = " ".join(output) + + output.append("{} second(s)".format(seconds)) + + output = " ".join(output) return output -def connect(args): - """ Create a connection from the given command line arguments. - """ - # If dbname looks like a conninfo string use it as such - if args.database and any(part in args.database for part in ['=', '://']): - return psycopg.connect(args.database) - - return psycopg.connect(dbname=args.database, user=args.username, - host=args.host, port=args.port) +def osm_date(date): + return date.strftime('%Y-%m-%dT%H:%M:%SZ') -def table_exists(conn, table_name, schema_name=None): - with conn.cursor() as cur: - if schema_name is not None: - cur.execute('SELECT * FROM pg_tables where tablename = %s and schemaname = %s ', (table_name, schema_name)) - else: - cur.execute('SELECT * FROM pg_tables where tablename = %s', (table_name, )) - return cur.rowcount > 0 +def from_osm_date(datestr): + return dt.datetime.strptime(datestr, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=dt.timezone.utc) -def compute_database_date(conn, schema, prefix): - """ Determine the date of the database from the newest object in the - database. - """ - # First, find the way with the highest ID in the database - # Using nodes would be more reliable but those are not cached by osm2pgsql. - with conn.cursor() as cur: - table = sql.Identifier(schema, f'{prefix}_ways') - cur.execute(sql.SQL("SELECT max(id) FROM {}").format(table)) - osmid = cur.fetchone()[0] if cur.rowcount == 1 else None - - if osmid is None: - LOG.fatal("No data found in the database.") - return None - - LOG.debug("Using way id %d for timestamp lookup", osmid) - # Get the way from the API to find the timestamp when it was created. - url = 'https://www.openstreetmap.org/api/0.6/way/{}/1'.format(osmid) - headers = {"User-Agent" : "osm2pgsql-update", - "Accept" : "application/json"} - with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response: - data = json.loads(response.read().decode('utf-8')) - - if not data.get('elements') or not 'timestamp' in data['elements'][0]: - LOG.fatal("The way data downloaded from the API does not contain valid data.\n" - "URL used: %s", url) - return None - date = data['elements'][0]['timestamp'] - LOG.debug("Found timestamp %s", date) +def start_point(param): + if param.isdigit(): + return int(param) - try: - date = dt.datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=dt.timezone.utc) - except ValueError: - LOG.fatal("Cannot parse timestamp '%s'", date) - return None + if sys.version_info >= (3, 7): + try: + date = dt.datetime.fromisoformat(param) + if date.tzinfo is None: + date = date.replace(tzinfo=dt.timezone.utc) + return date + except ValueError: + pass - return date.replace(tzinfo=dt.timezone.utc) + return from_osm_date(param) -def setup_replication_state(conn, table, base_url, seq, date): - """ (Re)create the table for the replication state and fill it with - the given state. - """ - with conn.cursor() as cur: - cur.execute(sql.SQL('DROP TABLE IF EXISTS {}').format(table)) - cur.execute(sql.SQL("""CREATE TABLE {} - (url TEXT, - sequence INTEGER, - importdate TIMESTAMP WITH TIME ZONE) - """).format(table)) - cur.execute(sql.SQL('INSERT INTO {} VALUES(%s, %s, %s)').format(table), - (base_url, seq, date)) - conn.commit() +class DBError(Exception): + def __init__(self, errno, msg): + self.errno = errno + self.msg = msg -def update_replication_state(conn, table, seq, date): - """ Update sequence and date in the replication state table. - The table is assumed to exist. - """ - with conn.cursor() as cur: - if date is not None: - cur.execute(sql.SQL('UPDATE {} SET sequence=%s, importdate=%s').format(table), - (seq, date)) + +class DBConnection: + + def __init__(self, args): + self.schema = args.middle_schema + + # If dbname looks like a conninfo string use it as such + if args.database and any(part in args.database for part in ['=', '://']): + self.conn = psycopg.connect(args.database, + fallback_application_name="osm2pgsql-replication") + else: + self.conn = psycopg.connect(dbname=args.database, user=args.username, + host=args.host, port=args.port, + fallback_application_name="osm2pgsql-replication") + + self.name = self.conn.get_dsn_parameters()['dbname'] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.conn is not None: + self.conn.close() + + def table_exists(self, table_name): + with self.conn.cursor() as cur: + cur.execute('SELECT * FROM pg_tables WHERE tablename = %s and schemaname = %s ', + (table_name, self.schema)) + return cur.rowcount > 0 + + def table_id(self, name): + return sql.Identifier(self.schema, name) + + +class Osm2pgsqlProperties: + PROP_TABLE_NAME = 'osm2pgsql_properties' + + def __init__(self, db): + self.db = db + self.is_updatable = self._get_prop('updatable') == 'true' + + def _get_prop(self, name): + with self.db.conn.cursor() as cur: + cur.execute(sql.SQL("SELECT value FROM {} WHERE property = %s") + .format(self.db.table_id(self.PROP_TABLE_NAME)), + (name, )) + return cur.fetchone()[0] if cur.rowcount == 1 else None + + def _set_prop(self, name, value): + with self.db.conn.cursor() as cur: + cur.execute(sql.SQL("""INSERT INTO {} (property, value) VALUES (%s, %s) + ON CONFLICT (property) + DO UPDATE SET value = EXCLUDED.value""") + .format(self.db.table_id(self.PROP_TABLE_NAME)), + (name, value)) + + def get_replication_base(self, server, start_at): + seq, date = None, None + if server is None: + server = self._get_prop('replication_base_url') + if server: + seq = self._get_prop('replication_sequence_number') + date = self._get_prop('replication_timestamp') + if date is not None: + date = from_osm_date(date) + else: + server = 'https://planet.openstreetmap.org/replication/minute' + + if isinstance(start_at, dt.datetime): + return server, None, start_at + + if seq is None or isinstance(start_at, int): + date = self._get_prop('current_timestamp') + if date is None: + raise DBError(1, "Cannot get timestamp from database. " + "Use --start-at to set an explicit date.") + + date = from_osm_date(date) + date -= dt.timedelta(minutes=start_at or 180) + seq = None else: - cur.execute(sql.SQL('UPDATE {} SET sequence=%s').format(table), - (seq,)) + seq = int(seq) + + return server, seq, date + + def get_replication_state(self): + if not self.db.table_exists(self.PROP_TABLE_NAME): + raise DBError(1, "Cannot find replication status table. Run 'osm2pgsql-replication init' first.") + + base_url = self._get_prop('replication_base_url') + seq = self._get_prop('replication_sequence_number') + date = self._get_prop('replication_timestamp') + + if base_url is None or seq is None or date is None: + raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-replication init' first.") + + return base_url, int(seq), from_osm_date(date) + + def write_replication_state(self, base_url, seq, date): + self._set_prop('replication_base_url', base_url) + self._set_prop('replication_sequence_number', seq) + self._set_prop('replication_timestamp', osm_date(date)) + self.db.conn.commit() + + +class LegacyProperties: + + def __init__(self, db, prefix): + self.db = db + self.prop_table = f'{prefix}_replication_status' + self.way_table = f'{prefix}_ways' + self.is_updatable = db.table_exists(self.way_table) + + def get_replication_base(self, server, start_at): + """ Determine the date of the database from the newest object in the + database. + """ + if server is None: + server = 'https://planet.openstreetmap.org/replication/minute' + + if isinstance(start_at, dt.datetime): + return server, None, start_at + + # First, find the way with the highest ID in the database + # Using nodes would be more reliable but those are not cached by osm2pgsql. + with self.db.conn.cursor() as cur: + cur.execute(sql.SQL("SELECT max(id) FROM {}") + .format(self.db.table_id(self.way_table))) + osmid = cur.fetchone()[0] if cur.rowcount == 1 else None + + if osmid is None: + raise DBError(1, "No data found in the database.") + + LOG.debug("Using way id %d for timestamp lookup", osmid) + # Get the way from the API to find the timestamp when it was created. + url = 'https://www.openstreetmap.org/api/0.6/way/{}/1'.format(osmid) + headers = {"User-Agent" : "osm2pgsql-replication", + "Accept" : "application/json"} + with urlrequest.urlopen(urlrequest.Request(url, headers=headers)) as response: + data = json.loads(response.read().decode('utf-8')) + + if not data.get('elements') or not 'timestamp' in data['elements'][0]: + raise DBError(1, "The way data downloaded from the API does not contain valid data.\n" + f"URL used: {url}") + + date = data['elements'][0]['timestamp'] + LOG.debug("Found timestamp %s", date) + + try: + date = from_osm_date(date) + except ValueError: + raise DBError(1, f"Cannot parse timestamp '{date}'") + + date -= dt.timedelta(minutes=start_at or 180) + + return server, None, date + + def get_replication_state(self): + if not self.db.table_exists(self.prop_table): + raise DBError(1, "Cannot find replication status table. Run 'osm2pgsql-replication init' first.") + + with self.db.conn.cursor() as cur: + cur.execute(sql.SQL('SELECT * FROM {}').format(self.db.table_id(self.prop_table))) + if cur.rowcount != 1: + raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-replication init' first.") + + base_url, seq, date = cur.fetchone() + + if base_url is None or seq is None or date is None: + raise DBError(2, "Updates not set up correctly. Run 'osm2pgsql-replication init' first.") + + return base_url, seq, date + + def write_replication_state(self, base_url, seq, date): + table = self.db.table_id(self.prop_table) + with self.db.conn.cursor() as cur: + if not self.db.table_exists(self.prop_table): + cur.execute(sql.SQL("""CREATE TABLE {} + (url TEXT, + sequence INTEGER, + importdate TIMESTAMP WITH TIME ZONE) + """).format(table)) + cur.execute(sql.SQL('TRUNCATE {}').format(table)) + if date: + cur.execute(sql.SQL('INSERT INTO {} VALUES(%s, %s, %s)').format(table), + (base_url, seq, date)) + else: + cur.execute(sql.SQL('INSERT INTO {} VALUES(%s, %s)').format(table), + (base_url, seq)) + self.db.conn.commit() - conn.commit() -def status(conn, args): +def get_status_info(props, args): + results = {'status': 0} + + base_url, db_seq, db_ts = props.get_replication_state() + + db_ts = db_ts.astimezone(dt.timezone.utc) + results['server'] = {'base_url': base_url} + results['local'] = {'sequence': db_seq, 'timestamp': osm_date(db_ts)} + + repl = ReplicationServer(base_url) + state_info = repl.get_state_info() + if state_info is None: + # PyOsmium was unable to download the state information + results['status'] = 3 + results['error'] = "Unable to download the state information from {}".format(base_url) + else: + results['status'] = 0 + now = dt.datetime.now(dt.timezone.utc) + + server_seq, server_ts = state_info + server_ts = server_ts.astimezone(dt.timezone.utc) + + results['server']['sequence'] = server_seq + results['server']['timestamp'] = osm_date(server_ts) + results['server']['age_sec'] = int((now-server_ts).total_seconds()) + + results['local']['age_sec'] = int((now - db_ts).total_seconds()) + + return results + + +def status(props, args): """\ Print information about the current replication status, optionally as JSON. @@ -206,47 +371,10 @@ `local` is the status of your server. """ - - results = {} - - if not table_exists(conn, args.table_name, args.middle_schema): - results['status'] = 1 - results['error'] = "Cannot find replication status table. Run 'osm2pgsql-replication init' first." - else: - with conn.cursor() as cur: - cur.execute(sql.SQL('SELECT * FROM {}').format(args.table)) - if cur.rowcount != 1: - results['status'] = 2 - results['error'] = "Updates not set up correctly. Run 'osm2pgsql-updates init' first." - else: - - base_url, db_seq, db_ts = cur.fetchone() - db_ts = db_ts.astimezone(dt.timezone.utc) - results['server'] = {} - results['local'] = {} - results['server']['base_url'] = base_url - results['local']['sequence'] = db_seq - results['local']['timestamp'] = db_ts.strftime("%Y-%m-%dT%H:%M:%SZ") - - - repl = ReplicationServer(base_url) - state_info = repl.get_state_info() - if state_info is None: - # PyOsmium was unable to download the state information - results['status'] = 3 - results['error'] = "Unable to download the state information from {}".format(base_url) - else: - results['status'] = 0 - now = dt.datetime.now(dt.timezone.utc) - - server_seq, server_ts = state_info - server_ts = server_ts.astimezone(dt.timezone.utc) - - results['server']['sequence'] = server_seq - results['server']['timestamp'] = server_ts.strftime("%Y-%m-%dT%H:%M:%SZ") - results['server']['age_sec'] = int((now-server_ts).total_seconds()) - - results['local']['age_sec'] = int((now - db_ts).total_seconds()) + try: + results = get_status_info(props, args) + except DBError as err: + results = {'status': err.errno, 'error': err.msg} if args.json: print(json.dumps(results)) @@ -268,66 +396,73 @@ print("Local database's most recent data is {} old".format(pretty_format_timedelta(results['local']['age_sec']))) - return results['status'] -def init(conn, args): +def init(props, args): """\ Initialise the replication process. - There are two ways to initialise the replication process: if you have imported - from a file that contains replication source information, then the - initialisation process can use this and set up replication from there. - Use the command `%(prog)s --osm-file ` for this. + This function sets the replication service to use and determines from + which date to apply updates. You must call this function at least once + to set up the replication process. It can safely be called again later + to change the replication servers or to roll back the update process and + reapply updates. + + There are different methods available for initialisation. When no + additional parameters are given, the data is initialised from the data + in the database. If the data was imported from a file with replication + information and the properties table is available (for osm2pgsql >= 1.9) + then the replication from the file is used. Otherwise the minutely + update service from openstreetmap.org is used as the default replication + service. The start date is either taken from the database timestamp + (for osm2pgsql >= 1.9) or determined from the newest way in the database + by querying the OSM API about its creation date. + + The replication service can be changed with the `--server` parameter. + To use a different start date, add `--start-at` with an absolute + ISO timestamp (e.g. 2007-08-20T12:21:53Z). When the program determines the + start date from the database timestamp or way creation date, then it + subtracts another 3 hours by default to ensure that all new changes are + available. To change this rollback period, use `--start-at` with the + number of minutes to rollback. This rollback mode can also be used to + force initialisation to use the database date and ignore the date + from the replication information in the file. - If the file has no replication information or you don't have the initial - import file anymore then replication can be set up according to - the data found in the database. It checks the planet_osm_way table for the - newest way in the database and then queries the OSM API when the way was - created. The date is used as the start date for replication. In this mode - the minutely diffs from the OSM servers are used as a source. You can change - this with the `--server` parameter. + The initialisation process can also use replication information from + an OSM file directly and ignore all other date information. + Use the command `%(prog)s --osm-file ` for this. """ if args.osm_file is None: - date = compute_database_date(conn, args.middle_schema, args.prefix) - if date is None: - return 1 - - date = date - dt.timedelta(hours=3) - base_url = args.server - seq = None + base_url, seq, date = props.get_replication_base(args.server, args.start_at) else: base_url, seq, date = get_replication_header(args.osm_file) if base_url is None or (seq is None and date is None): - LOG.fatal("File '%s' has no usable replication headers. Use '--server' instead.") - return 1 + raise DBError(1, f"File '{args.osm_file}' has no usable replication headers. Use '--server' instead.") repl = ReplicationServer(base_url) if seq is None: seq = repl.timestamp_to_sequence(date) - if seq is None: - LOG.fatal("Cannot reach the configured replication service '%s'.\n" - "Does the URL point to a directory containing OSM update data?", - base_url) - return 1 + raise DBError(1, f"Cannot reach the configured replication service '{base_url}'.\n" + "Does the URL point to a directory containing OSM update data?") if date is None: state = repl.get_state_info(seq) if state is None: - LOG.fatal("Cannot reach the configured replication service '%s'.\n" - "Does the URL point to a directory containing OSM update data?", - base_url) + raise DBError(1, f"Cannot reach the configured replication service '{base_url}'.\n" + "Does the URL point to a directory containing OSM update data?") + date = from_osm_date(state.timestamp) - setup_replication_state(conn, args.table, base_url, seq, date) + props.write_replication_state(base_url, seq, date) LOG.info("Initialised updates for service '%s'.", base_url) - LOG.info("Starting at sequence %d (%s).", seq, date) + LOG.info("Starting at sequence %d (%s).", seq, osm_date(date)) return 0 -def update(conn, args): + +def update(props, args): """\ Download newly available data and apply it to the database. @@ -353,28 +488,33 @@ may be missing in the rare case that the replication service stops responding after the updates have been downloaded. """ - if not table_exists(conn, args.table_name, args.middle_schema): - LOG.fatal("Cannot find replication status table. " - "Run 'osm2pgsql-replication init' first.") - return 1 - - with conn.cursor() as cur: - cur.execute(sql.SQL('SELECT * FROM {}').format(args.table)) - if cur.rowcount != 1: - LOG.fatal("Updates not set up correctly. Run 'osm2pgsql-updates init' first.") - return 1 + base_url, seq, ts = props.get_replication_state() - base_url, seq, ts = cur.fetchone() - LOG.info("Using replication service '%s'. Current sequence %d (%s).", - base_url, seq, ts) + initial_local_timestamp = ts + LOG.info("Using replication service '%s'.", base_url) + local_db_age_sec = int((dt.datetime.now(dt.timezone.utc) - ts).total_seconds()) repl = ReplicationServer(base_url) current = repl.get_state_info() + if current is None: + raise DBError(1, f"Cannot reach the configured replication service '{base_url}'.\n" + "Does the URL point to a directory containing OSM update data?") if seq >= current.sequence: LOG.info("Database already up-to-date.") return 0 + remote_server_age_sec = int((dt.datetime.now(dt.timezone.utc) - current.timestamp).total_seconds()) + LOG.debug("Applying %d sequence(s) (%d → %d), covering %s (%s sec) of changes (%s → %s)", + current.sequence - seq, current.sequence, seq, + pretty_format_timedelta((current.timestamp - ts).total_seconds()), + int((current.timestamp - ts).total_seconds()), + osm_date(ts.astimezone(dt.timezone.utc)), + osm_date(current.timestamp.astimezone(dt.timezone.utc)) + ) + + update_started = dt.datetime.now(dt.timezone.utc) + if args.diff_file is not None: outfile = Path(args.diff_file) else: @@ -420,19 +560,30 @@ LOG.debug('Calling post-processing script: %s', ' '.join(cmd)) subprocess.run(cmd, check=True) - update_replication_state(conn, args.table, seq, - nextstate.timestamp if nextstate else None) + props.write_replication_state(base_url, seq, nextstate.timestamp if nextstate else None) if nextstate is not None: LOG.info("Data imported until %s. Backlog remaining: %s", - nextstate.timestamp, - dt.datetime.now(dt.timezone.utc) - nextstate.timestamp) + osm_date(nextstate.timestamp.astimezone(dt.timezone.utc)), + pretty_format_timedelta((dt.datetime.now(dt.timezone.utc) - nextstate.timestamp).total_seconds()), + ) if args.once: break + update_duration_sec = (dt.datetime.now(dt.timezone.utc) - update_started).total_seconds() + _base_url, _seq, current_local_timestamp = props.get_replication_state() + + total_applied_changes_duration_sec = (current_local_timestamp - initial_local_timestamp).total_seconds() + LOG.debug("It took %s (%d sec) to apply %s (%d sec) of changes. This is a speed of ×%.1f.", + pretty_format_timedelta(update_duration_sec), int(update_duration_sec), + pretty_format_timedelta(total_applied_changes_duration_sec), int(total_applied_changes_duration_sec), + total_applied_changes_duration_sec / update_duration_sec + ) + return 0 + def get_parser(): parser = ArgumentParser(description=__doc__, prog='osm2pgsql-replication', @@ -464,8 +615,10 @@ help='Database server port') group.add_argument('-p', '--prefix', metavar='PREFIX', default='planet_osm', help="Prefix for table names (default 'planet_osm')") - group.add_argument('--middle-schema', metavar='MIDDLE_SCHEMA', default='public', + group.add_argument('--middle-schema', metavar='SCHEMA', default=None, help='Name of the schema to store the table for the replication state in') + group.add_argument('--schema', metavar='SCHEMA', default=None, + help='Name of the schema for the database') # Arguments for init cmd = subs.add_parser('init', parents=[default_args], @@ -478,8 +631,13 @@ srcgrp.add_argument('--osm-file', metavar='FILE', help='Get replication information from the given file.') srcgrp.add_argument('--server', metavar='URL', - default='https://planet.openstreetmap.org/replication/minute', - help='Use replication server at the given URL (default: %(default)s)') + help='Use replication server at the given URL') + grp.add_argument('--start-at', metavar='TIME', type=start_point, + help='Time when to start replication. When an absolute timestamp ' + '(in ISO format) is given, it will be used. If a number ' + 'is given, then replication starts the number of minutes ' + 'before the known date of the database.') + cmd.set_defaults(handler=init) # Arguments for update @@ -513,12 +671,15 @@ cmd.add_argument('--json', action="store_true", default=False, help="Output status as json.") cmd.set_defaults(handler=status) - return parser -def main(): + +def main(prog_args=None): parser = get_parser() - args = parser.parse_args() + try: + args = parser.parse_args(args=prog_args) + except SystemExit: + return 1 if missing_modules: LOG.fatal("Missing required Python libraries %(mods)s.\n\n" @@ -538,20 +699,25 @@ datefmt='%Y-%m-%d %H:%M:%S', level=max(4 - args.verbose, 1) * 10) - args.table_name = f'{args.prefix}_replication_status' - args.table = sql.Identifier(args.middle_schema, args.table_name) + args.middle_schema = args.middle_schema or args.schema or 'public' - conn = connect(args) + with DBConnection(args) as db: + if db.table_exists(Osm2pgsqlProperties.PROP_TABLE_NAME): + props = Osm2pgsqlProperties(db) + else: + props = LegacyProperties(db, args.prefix) - try: - if not table_exists(conn, f'{args.prefix}_ways'): - LOG.fatal(f'osm2pgsql middle table "{args.prefix}_ways" not found in database "{args.database}". ' + if not props.is_updatable: + LOG.fatal(f'osm2pgsql middle table "{args.middle_schema}.{args.prefix}_ways" not found in database "{db.name}". ' 'Database needs to be imported in --slim mode.') return 1 - return args.handler(conn, args) - finally: - conn.close() + try: + return args.handler(props, args) + except DBError as err: + LOG.fatal(err.msg) + + return 1 if __name__ == '__main__': diff -Nru osm2pgsql-1.8.1+ds/src/CMakeLists.txt osm2pgsql-1.9.0+ds/src/CMakeLists.txt --- osm2pgsql-1.8.1+ds/src/CMakeLists.txt 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/CMakeLists.txt 2023-08-15 19:18:18.000000000 +0000 @@ -1,10 +1,17 @@ add_library(osm2pgsql_lib STATIC) +# Set the minimum required C++ version for the library and hence for all +# binaries that use it. +target_compile_features(osm2pgsql_lib PUBLIC cxx_std_17) + target_sources(osm2pgsql_lib PRIVATE + command-line-parser.cpp db-copy.cpp + debug-output.cpp dependency-manager.cpp expire-tiles.cpp + expire-output.cpp gazetteer-style.cpp geom.cpp geom-box.cpp @@ -18,7 +25,6 @@ middle-ram.cpp node-locations.cpp node-persistent-cache.cpp - options.cpp ordered-index.cpp osmdata.cpp output-gazetteer.cpp @@ -29,6 +35,7 @@ pgsql-capabilities.cpp pgsql-helper.cpp progress-display.cpp + properties.cpp reprojection.cpp table.cpp taginfo.cpp @@ -50,11 +57,13 @@ flex-index.cpp flex-table.cpp flex-table-column.cpp + flex-lua-expire-output.cpp flex-lua-geom.cpp flex-lua-index.cpp flex-lua-table.cpp flex-write.cpp geom-transform.cpp + lua-setup.cpp lua-utils.cpp output-flex.cpp tagtransform-lua.cpp diff -Nru osm2pgsql-1.8.1+ds/src/command-line-parser.cpp osm2pgsql-1.9.0+ds/src/command-line-parser.cpp --- osm2pgsql-1.8.1+ds/src/command-line-parser.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/command-line-parser.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,819 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "command-line-parser.hpp" + +#include "format.hpp" +#include "logging.hpp" +#include "options.hpp" +#include "pgsql.hpp" +#include "reprojection.hpp" +#include "util.hpp" +#include "version.hpp" + +#include + +#include + +#ifdef HAVE_LUA +#include +#endif + +#include +#include +#include +#include +#include // for number of threads + +static char const *program_name(char const *name) +{ + char const *const slash = std::strrchr(name, '/'); + return slash ? (slash + 1) : name; +} + +namespace { +char const *const short_options = + "ab:cd:KhlmMp:suvU:WH:P:i:IE:C:S:e:o:O:xkjGz:r:VF:"; + +struct option const long_options[] = { + {"append", no_argument, nullptr, 'a'}, + {"bbox", required_argument, nullptr, 'b'}, + {"cache", required_argument, nullptr, 'C'}, + {"cache-strategy", required_argument, nullptr, 204}, + {"create", no_argument, nullptr, 'c'}, + {"database", required_argument, nullptr, 'd'}, + {"disable-parallel-indexing", no_argument, nullptr, 'I'}, + {"drop", no_argument, nullptr, 206}, + {"expire-bbox-size", required_argument, nullptr, 214}, + {"expire-output", required_argument, nullptr, 'o'}, + {"expire-tiles", required_argument, nullptr, 'e'}, + {"extra-attributes", no_argument, nullptr, 'x'}, + {"flat-nodes", required_argument, nullptr, 'F'}, + {"help", no_argument, nullptr, 'h'}, + {"host", required_argument, nullptr, 'H'}, + {"hstore", no_argument, nullptr, 'k'}, + {"hstore-add-index", no_argument, nullptr, 211}, + {"hstore-all", no_argument, nullptr, 'j'}, + {"hstore-column", required_argument, nullptr, 'z'}, + {"hstore-match-only", no_argument, nullptr, 208}, + {"input-reader", required_argument, nullptr, 'r'}, + {"keep-coastlines", no_argument, nullptr, 'K'}, + {"latlong", no_argument, nullptr, 'l'}, + {"log-level", required_argument, nullptr, 400}, + {"log-progress", required_argument, nullptr, 401}, + {"log-sql", no_argument, nullptr, 402}, + {"log-sql-data", no_argument, nullptr, 403}, + {"merc", no_argument, nullptr, 'm'}, + {"middle-schema", required_argument, nullptr, 215}, + {"middle-way-node-index-id-shift", required_argument, nullptr, 300}, + {"middle-database-format", required_argument, nullptr, 301}, + {"middle-with-nodes", no_argument, nullptr, 302}, + {"multi-geometry", no_argument, nullptr, 'G'}, + {"number-processes", required_argument, nullptr, 205}, + {"output", required_argument, nullptr, 'O'}, + {"output-pgsql-schema", required_argument, nullptr, 216}, + {"password", no_argument, nullptr, 'W'}, + {"port", required_argument, nullptr, 'P'}, + {"prefix", required_argument, nullptr, 'p'}, + {"proj", required_argument, nullptr, 'E'}, + {"reproject-area", no_argument, nullptr, 213}, + {"schema", required_argument, nullptr, 218}, + {"slim", no_argument, nullptr, 's'}, + {"style", required_argument, nullptr, 'S'}, + {"tablespace-index", required_argument, nullptr, 'i'}, + {"tablespace-main-data", required_argument, nullptr, 202}, + {"tablespace-main-index", required_argument, nullptr, 203}, + {"tablespace-slim-data", required_argument, nullptr, 200}, + {"tablespace-slim-index", required_argument, nullptr, 201}, + {"tag-transform-script", required_argument, nullptr, 212}, + {"username", required_argument, nullptr, 'U'}, + {"verbose", no_argument, nullptr, 'v'}, + {"version", no_argument, nullptr, 'V'}, + {"with-forward-dependencies", required_argument, nullptr, 217}, + {nullptr, 0, nullptr, 0}}; + +} // anonymous namespace + +void long_usage(char const *arg0, bool verbose) +{ + char const *const name = program_name(arg0); + + fmt::print(stdout, "\nUsage: {} [OPTIONS] OSM-FILE...\n", name); + (void)std::fputs( + "\nImport data from the OSM file(s) into a PostgreSQL database.\n\n\ +Full documentation is available at https://osm2pgsql.org/\n\n", + stdout); + + (void)std::fputs("\ +Common options:\n\ + -a|--append Update existing osm2pgsql database with data from file.\n\ + -c|--create Import OSM data from file into database. This is the\n\ + default if --append is not specified.\n\ + -O|--output=OUTPUT Set output. Options are:\n\ + pgsql - Output to a PostGIS database (default)\n\ + flex - More flexible output to PostGIS database\n\ + gazetteer - Output to a PostGIS database for Nominatim\n\ + (deprecated)\n\ + null - No output. Used for testing.\n\ + -S|--style=FILE Location of the style file. Defaults to\n\ + '" DEFAULT_STYLE "'.\n\ + -k|--hstore Add tags without column to an additional hstore column.\n", + stdout); +#ifdef HAVE_LUA + (void)std::fputs("\ + --tag-transform-script=SCRIPT Specify a Lua script to handle tag\n\ + filtering and normalisation (pgsql output only).\n", + stdout); +#endif + (void)std::fputs("\ + -s|--slim Store temporary data in the database. This switch is\n\ + required if you want to update with --append later.\n\ + --drop Only with --slim: drop temporary tables after import\n\ + (no updates are possible).\n\ + -C|--cache=SIZE Use up to SIZE MB for caching nodes (default: 800).\n\ + -F|--flat-nodes=FILE Specifies the file to use to persistently store node\n\ + information in slim mode instead of in PostgreSQL.\n\ + This is a single large file (> 50GB). Only recommended\n\ + for full planet imports. Default is disabled.\n\ + --schema=SCHEMA Default schema (default: 'public').\n\ +\n\ +Database options:\n\ + -d|--database=DB The name of the PostgreSQL database to connect to or\n\ + a PostgreSQL conninfo string.\n\ + -U|--username=NAME PostgreSQL user name.\n\ + -W|--password Force password prompt.\n\ + -H|--host=HOST Database server host name or socket location.\n\ + -P|--port=PORT Database server port.\n", + stdout); + + if (verbose) { + (void)std::fputs("\n\ +Logging options:\n\ + --log-level=LEVEL Set log level ('debug', 'info' (default), 'warn',\n\ + or 'error').\n\ + --log-progress=VALUE Enable ('true') or disable ('false') progress\n\ + logging. If set to 'auto' osm2pgsql will enable progress\n\ + logging on the console and disable it if the output is\n\ + redirected to a file. Default: true.\n\ + --log-sql Enable logging of SQL commands for debugging.\n\ + --log-sql-data Enable logging of all data added to the database.\n\ + -v|--verbose Same as '--log-level=debug'.\n\ +\n\ +Input options:\n\ + -r|--input-reader=FORMAT Input format ('xml', 'pbf', 'o5m', or\n\ + 'auto' - autodetect format (default))\n\ + -b|--bbox=MINLON,MINLAT,MAXLON,MAXLAT Apply a bounding box filter on the\n\ + imported data, e.g. '--bbox -0.5,51.25,0.5,51.75'.\n\ +\n\ +Middle options:\n\ + -i|--tablespace-index=TBLSPC The name of the PostgreSQL tablespace where\n\ + all indexes will be created.\n\ + The following options allow more fine-grained control:\n\ + --tablespace-slim-data=TBLSPC Tablespace for slim mode tables.\n\ + --tablespace-slim-index=TBLSPC Tablespace for slim mode indexes.\n\ + (if unset, use db's default; -i is equivalent to setting\n\ + --tablespace-main-index and --tablespace-slim-index).\n\ + -p|--prefix=PREFIX Prefix for table names (default 'planet_osm')\n\ + --cache-strategy=STRATEGY Deprecated. Not used any more.\n\ + -x|--extra-attributes Include attributes (user name, user id, changeset\n\ + id, timestamp and version) for each object in the database.\n\ + --middle-schema=SCHEMA Schema to use for middle tables (default: setting of --schema).\n\ + --middle-way-node-index-id-shift=SHIFT Set ID shift for bucket index.\n\ + --middle-database-format=FORMAT Set middle db format (default: legacy).\n\ + --middle-with-nodes Store tagged nodes in db (new middle db format only).\n\ +\n\ +Pgsql output options:\n\ + -i|--tablespace-index=TBLSPC The name of the PostgreSQL tablespace where\n\ + all indexes will be created.\n\ + The following options allow more fine-grained control:\n\ + --tablespace-main-data=TBLSPC Tablespace for main tables.\n\ + --tablespace-main-index=TBLSPC Tablespace for main table indexes.\n\ + -l|--latlong Store data in degrees of latitude & longitude (WGS84).\n\ + -m|--merc Store data in web mercator (default).\n" +#ifdef HAVE_GENERIC_PROJ + " -E|--proj=SRID Use projection EPSG:SRID.\n" +#endif + "\ + -p|--prefix=PREFIX Prefix for table names (default 'planet_osm').\n\ + -x|--extra-attributes Include attributes (user name, user id, changeset\n\ + id, timestamp and version) for each object in the database.\n\ + --hstore-match-only Only keep objects that have a value in one of the\n\ + columns (default with --hstore is to keep all objects).\n\ + -j|--hstore-all Add all tags to an additional hstore (key/value) column.\n\ + -z|--hstore-column=NAME Add an additional hstore (key/value) column\n\ + containing all tags that start with the specified string,\n\ + eg '--hstore-column name:' will produce an extra hstore\n\ + column that contains all 'name:xx' tags.\n\ + --hstore-add-index Add index to hstore column.\n\ + -G|--multi-geometry Generate multi-geometry features in postgresql tables.\n\ + -K|--keep-coastlines Keep coastline data rather than filtering it out.\n\ + Default: discard objects tagged natural=coastline.\n\ + --output-pgsql-schema=SCHEMA Schema to use for pgsql output tables\n\ + (default: setting of --schema).\n\ + --reproject-area Compute area column using web mercator coordinates.\n\ +\n\ +Expiry options:\n\ + -e|--expire-tiles=[MIN_ZOOM-]MAX_ZOOM Create a tile expiry list.\n\ + Zoom levels must be larger than 0 and smaller than 32.\n\ + -o|--expire-output=FILENAME Output filename for expired tiles list.\n\ + --expire-bbox-size=SIZE Max size for a polygon to expire the whole\n\ + polygon, not just the boundary.\n\ +\n\ +Advanced options:\n\ + -I|--disable-parallel-indexing Disable indexing all tables concurrently.\n\ + --number-processes=NUM Specifies the number of parallel processes used\n\ + for certain operations (default depends on number of CPUs).\n\ + --with-forward-dependencies=BOOL Propagate changes from nodes to ways\n\ + and node/way members to relations (Default: true).\n\ +", + stdout); + } else { + fmt::print( + stdout, + "\nRun '{} --help --verbose' (-h -v) for a full list of options.\n", + name); + } +} + +static bool compare_prefix(std::string const &str, + std::string const &prefix) noexcept +{ + return std::strncmp(str.c_str(), prefix.c_str(), prefix.size()) == 0; +} + +std::string build_conninfo(database_options_t const &opt) +{ + if (compare_prefix(opt.db, "postgresql://") || + compare_prefix(opt.db, "postgres://")) { + return opt.db; + } + + util::string_joiner_t joiner{' '}; + joiner.add("fallback_application_name='osm2pgsql'"); + + if (std::strchr(opt.db.c_str(), '=') != nullptr) { + joiner.add(opt.db); + return joiner(); + } + + joiner.add("client_encoding='UTF8'"); + + if (!opt.db.empty()) { + joiner.add(fmt::format("dbname='{}'", opt.db)); + } + if (!opt.username.empty()) { + joiner.add(fmt::format("user='{}'", opt.username)); + } + if (!opt.password.empty()) { + joiner.add(fmt::format("password='{}'", opt.password)); + } + if (!opt.host.empty()) { + joiner.add(fmt::format("host='{}'", opt.host)); + } + if (!opt.port.empty()) { + joiner.add(fmt::format("port='{}'", opt.port)); + } + + return joiner(); +} + +static osmium::Box parse_bbox_param(char const *arg) +{ + double minx = NAN; + double maxx = NAN; + double miny = NAN; + double maxy = NAN; + + int const n = sscanf(arg, "%lf,%lf,%lf,%lf", &minx, &miny, &maxx, &maxy); + if (n != 4) { + throw std::runtime_error{"Bounding box must be specified like: " + "minlon,minlat,maxlon,maxlat."}; + } + + if (maxx <= minx) { + throw std::runtime_error{ + "Bounding box failed due to maxlon <= minlon."}; + } + + if (maxy <= miny) { + throw std::runtime_error{ + "Bounding box failed due to maxlat <= minlat."}; + } + + log_debug("Applying bounding box: {},{} to {},{}", minx, miny, maxx, maxy); + + return osmium::Box{minx, miny, maxx, maxy}; +} + +static unsigned int parse_number_processes_param(char const *arg) +{ + int num = atoi(arg); + if (num < 1) { + log_warn("--number-processes must be at least 1. Using 1."); + num = 1; + } else if (num > 32) { + // The threads will open up database connections which will + // run out at some point. It depends on the number of tables + // how many connections there are. The number 32 is way beyond + // anything that will make sense here. + log_warn("--number-processes too large. Set to 32."); + num = 32; + } + + return static_cast(num); +} + +static void parse_expire_tiles_param(char const *arg, + uint32_t *expire_tiles_zoom_min, + uint32_t *expire_tiles_zoom) +{ + if (!arg || arg[0] == '-') { + throw std::runtime_error{"Missing argument for option --expire-tiles." + " Zoom levels must be positive."}; + } + + char *next_char = nullptr; + *expire_tiles_zoom_min = + static_cast(std::strtoul(arg, &next_char, 10)); + + if (*expire_tiles_zoom_min == 0) { + throw std::runtime_error{"Bad argument for option --expire-tiles." + " Minimum zoom level must be larger than 0."}; + } + + // The first character after the number is ignored because that is the + // separating hyphen. + if (*next_char == '-') { + ++next_char; + // Second number must not be negative because zoom levels must be + // positive. + if (next_char && *next_char != '-' && isdigit(*next_char)) { + char *after_maxzoom = nullptr; + *expire_tiles_zoom = static_cast( + std::strtoul(next_char, &after_maxzoom, 10)); + if (*expire_tiles_zoom == 0 || *after_maxzoom != '\0') { + throw std::runtime_error{"Invalid maximum zoom level" + " given for tile expiry."}; + } + } else { + throw std::runtime_error{ + "Invalid maximum zoom level given for tile expiry."}; + } + return; + } + + if (*next_char == '\0') { + // end of string, no second zoom level given + *expire_tiles_zoom = *expire_tiles_zoom_min; + return; + } + + throw std::runtime_error{"Minimum and maximum zoom level for" + " tile expiry must be separated by '-'."}; +} + +static void parse_log_level_param(char const *arg) +{ + if (std::strcmp(arg, "debug") == 0) { + get_logger().set_level(log_level::debug); + } else if (std::strcmp(arg, "info") == 0) { + get_logger().set_level(log_level::info); + } else if ((std::strcmp(arg, "warn") == 0) || + (std::strcmp(arg, "warning") == 0)) { + get_logger().set_level(log_level::warn); + } else if (std::strcmp(arg, "error") == 0) { + get_logger().set_level(log_level::error); + } else { + throw fmt_error("Unknown value for --log-level option: {}", arg); + } +} + +static void parse_log_progress_param(char const *arg) +{ + if (std::strcmp(arg, "true") == 0) { + get_logger().enable_progress(); + } else if (std::strcmp(arg, "false") == 0) { + get_logger().disable_progress(); + } else if (std::strcmp(arg, "auto") == 0) { + get_logger().auto_progress(); + } else { + throw fmt_error("Unknown value for --log-progress option: {}", arg); + } +} + +static bool parse_with_forward_dependencies_param(char const *arg) +{ + log_warn("The option --with-forward-dependencies is deprecated and will " + "soon be removed."); + + if (std::strcmp(arg, "false") == 0) { + return false; + } + + if (std::strcmp(arg, "true") == 0) { + return true; + } + + throw fmt_error("Unknown value for --with-forward-dependencies option: {}", + arg); +} + +void print_version() +{ + fmt::print(stderr, "Build: {}\n", get_build_type()); + fmt::print(stderr, "Compiled using the following library versions:\n"); + fmt::print(stderr, "Libosmium {}\n", LIBOSMIUM_VERSION_STRING); + fmt::print(stderr, "Proj {}\n", get_proj_version()); +#ifdef HAVE_LUA +#ifdef HAVE_LUAJIT + fmt::print(stderr, "{} ({})\n", LUA_RELEASE, LUAJIT_VERSION); +#else + fmt::print(stderr, "{}\n", LUA_RELEASE); +#endif +#else + fmt::print(stderr, "Lua support not included\n"); +#endif +} + +static void check_options(options_t *options) +{ + if (options->append && options->create) { + throw std::runtime_error{"--append and --create options can not be " + "used at the same time!"}; + } + + if (options->append && !options->slim) { + throw std::runtime_error{"--append can only be used with slim mode!"}; + } + + if (options->droptemp && !options->slim) { + throw std::runtime_error{"--drop only makes sense with --slim."}; + } + + if (options->append && options->middle_database_format != 1) { + throw std::runtime_error{ + "Do not use --middle-database-format with --append."}; + } + + if (options->hstore_mode == hstore_column::none && + options->hstore_columns.empty() && options->hstore_match_only) { + log_warn("--hstore-match-only only makes sense with --hstore, " + "--hstore-all, or --hstore-column; ignored."); + options->hstore_match_only = false; + } + + if (options->enable_hstore_index && + options->hstore_mode == hstore_column::none && + options->hstore_columns.empty()) { + log_warn("--hstore-add-index only makes sense with hstore enabled; " + "ignored."); + options->enable_hstore_index = false; + } + + if (options->cache < 0) { + options->cache = 0; + log_warn("RAM cache cannot be negative. Using 0 instead."); + } + + if (options->cache == 0) { + if (!options->slim) { + throw std::runtime_error{ + "RAM node cache can only be disabled in slim mode."}; + } + if (options->flat_node_file.empty()) { + log_warn("RAM cache is disabled. This will likely slow down " + "processing a lot."); + } + } + + if (!options->slim && !options->flat_node_file.empty()) { + log_warn("Ignoring --flat-nodes/-F setting in non-slim mode"); + } + + // zoom level 31 is the technical limit because we use 32-bit integers for the x and y index of a tile ID + if (options->expire_tiles_zoom_min > 31) { + options->expire_tiles_zoom_min = 31; + log_warn("Minimum zoom level for tile expiry is too " + "large and has been set to 31."); + } + + if (options->expire_tiles_zoom > 31) { + options->expire_tiles_zoom = 31; + log_warn("Maximum zoom level for tile expiry is too " + "large and has been set to 31."); + } + + if (options->expire_tiles_zoom != 0 && + options->projection->target_srs() != 3857) { + log_warn("Expire has been enabled (with -e or --expire-tiles) but " + "target SRS is not Mercator (EPSG:3857). Expire disabled!"); + options->expire_tiles_zoom = 0; + } + + if (options->output_backend == "gazetteer") { + log_warn( + "The 'gazetteer' output is deprecated and will soon be removed."); + } +} + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +options_t parse_command_line(int argc, char *argv[]) +{ + options_t options; + + options.num_procs = std::min(4U, std::thread::hardware_concurrency()); + if (options.num_procs < 1) { + log_warn("Unable to detect number of hardware threads supported!" + " Using single thread."); + options.num_procs = 1; + } + + // If there are no command line arguments at all, show help. + if (argc == 1) { + options.command = command_t::help; + long_usage(argv[0], false); + return options; + } + + database_options_t database_options; + + bool print_help = false; + bool help_verbose = false; // Will be set when -v/--verbose is set + + int c = 0; + + //keep going while there are args left to handle + // note: optind would seem to need to be set to 1, but that gives valgrind + // errors - setting it to zero seems to work, though. see + // http://stackoverflow.com/questions/15179963/is-it-possible-to-repeat-getopt#15179990 + optind = 0; + // NOLINTNEXTLINE(concurrency-mt-unsafe) + while (-1 != (c = getopt_long(argc, argv, short_options, long_options, + nullptr))) { + + //handle the current arg + switch (c) { + case 'a': // --append + options.append = true; + break; + case 'b': // --bbox + options.bbox = parse_bbox_param(optarg); + break; + case 'c': // --create + options.create = true; + break; + case 'v': // --verbose + help_verbose = true; + get_logger().set_level(log_level::debug); + break; + case 's': // --slim + options.slim = true; + break; + case 'K': // --keep-coastlines + options.keep_coastlines = true; + break; + case 'l': // --latlong + options.projection = reprojection::create_projection(PROJ_LATLONG); + break; + case 'm': // --merc + options.projection = + reprojection::create_projection(PROJ_SPHERE_MERC); + break; + case 'E': // --proj +#ifdef HAVE_GENERIC_PROJ + options.projection = reprojection::create_projection(atoi(optarg)); +#else + throw std::runtime_error{"Generic projections not available."}; +#endif + break; + case 'p': // --prefix + options.prefix = optarg; + options.prefix_is_set = true; + check_identifier(options.prefix, "--prefix parameter"); + break; + case 'd': // --database + database_options.db = optarg; + break; + case 'C': // --cache + options.cache = atoi(optarg); + break; + case 'U': // --username + database_options.username = optarg; + break; + case 'W': // --password + options.pass_prompt = true; + break; + case 'H': // --host + database_options.host = optarg; + break; + case 'P': // --port + database_options.port = optarg; + break; + case 'S': // --style + options.style = optarg; + options.style_set = true; + break; + case 'i': // --tablespace-index + options.tblsmain_index = optarg; + options.tblsslim_index = options.tblsmain_index; + break; + case 200: // --tablespace-slim-data + options.tblsslim_data = optarg; + break; + case 201: // --tablespace-slim-index + options.tblsslim_index = optarg; + break; + case 202: // --tablespace-main-data + options.tblsmain_data = optarg; + break; + case 203: // --tablespace-main-index + options.tblsmain_index = optarg; + break; + case 'e': // --expire-tiles + parse_expire_tiles_param(optarg, &options.expire_tiles_zoom_min, + &options.expire_tiles_zoom); + break; + case 'o': // --expire-output + options.expire_tiles_filename = optarg; + break; + case 214: // --expire-bbox-size + options.expire_tiles_max_bbox = atof(optarg); + break; + case 'O': // --output + options.output_backend = optarg; + options.output_backend_set = true; + break; + case 'x': // --extra-attributes + options.extra_attributes = true; + break; + case 'k': // --hstore + if (options.hstore_mode != hstore_column::none) { + throw std::runtime_error{"You can not specify both --hstore " + "(-k) and --hstore-all (-j)."}; + } + options.hstore_mode = hstore_column::norm; + break; + case 208: // --hstore-match-only + options.hstore_match_only = true; + break; + case 'j': // --hstore-all + if (options.hstore_mode != hstore_column::none) { + throw std::runtime_error{"You can not specify both --hstore " + "(-k) and --hstore-all (-j)."}; + } + options.hstore_mode = hstore_column::all; + break; + case 'z': // --hstore-column + options.hstore_columns.emplace_back(optarg); + break; + case 'G': // --multi-geometry + options.enable_multi = true; + break; + case 'r': // --input-reader + if (std::strcmp(optarg, "auto") != 0) { + options.input_format = optarg; + } + break; + case 'h': // --help + print_help = true; + break; + case 'I': // --disable-parallel-indexing + options.parallel_indexing = false; + break; + case 204: // -cache-strategy + log_warn("Deprecated option --cache-strategy ignored"); + break; + case 205: // --number-processes + options.num_procs = parse_number_processes_param(optarg); + break; + case 206: // --drop + options.droptemp = true; + break; + case 'F': // --flat-nodes + options.flat_node_file = optarg; + break; + case 211: // --hstore-add-index + options.enable_hstore_index = true; + break; + case 212: // --tag-transform-script + options.tag_transform_script = optarg; + break; + case 213: // --reproject-area + options.reproject_area = true; + break; + case 'V': // --version + options.command = command_t::version; + return options; + case 215: // --middle-schema + options.middle_dbschema = optarg; + if (options.middle_dbschema.empty()) { + throw std::runtime_error{"Schema can not be empty."}; + } + check_identifier(options.middle_dbschema, + "--middle-schema parameter"); + break; + case 216: // --output-pgsql-schema + options.output_dbschema = optarg; + if (options.output_dbschema.empty()) { + throw std::runtime_error{"Schema can not be empty."}; + } + check_identifier(options.output_dbschema, + "--output-pgsql-schema parameter"); + break; + case 217: // --with-forward-dependencies=BOOL + options.with_forward_dependencies = + parse_with_forward_dependencies_param(optarg); + break; + case 218: // --schema + options.dbschema = optarg; + if (options.dbschema.empty()) { + throw std::runtime_error{"Schema can not be empty."}; + } + check_identifier(options.dbschema, "--schema parameter"); + break; + case 300: // --middle-way-node-index-id-shift + options.way_node_index_id_shift = atoi(optarg); + break; + case 301: // --middle-database-format + if (optarg == std::string{"legacy"}) { + options.middle_database_format = 1; + } else if (optarg == std::string{"new"}) { + options.middle_database_format = 2; + } else { + throw std::runtime_error{ + "Unknown value for --middle-database-format (Use 'legacy' " + "or 'new')."}; + } + break; + case 302: // --middle-with-nodes + options.middle_with_nodes = true; + break; + case 400: // --log-level=LEVEL + parse_log_level_param(optarg); + break; + case 401: // --log-progress=VALUE + parse_log_progress_param(optarg); + break; + case 402: // --log-sql + get_logger().enable_sql(); + break; + case 403: // --log-sql-data + get_logger().enable_sql_data(); + break; + case '?': + default: + throw std::runtime_error{"Usage error. Try 'osm2pgsql --help'."}; + } + } //end while + + if (options.middle_dbschema.empty()) { + options.middle_dbschema = options.dbschema; + } + + if (options.output_dbschema.empty()) { + options.output_dbschema = options.dbschema; + } + + //they were looking for usage info + if (print_help) { + options.command = command_t::help; + long_usage(argv[0], help_verbose); + return options; + } + + //we require some input files! + if (optind >= argc) { + throw std::runtime_error{ + "Missing input file(s). Try 'osm2pgsql --help'."}; + } + + //get the input files + while (optind < argc) { + options.input_files.emplace_back(argv[optind]); + ++optind; + } + + if (!options.projection) { + options.projection = reprojection::create_projection(PROJ_SPHERE_MERC); + } + + check_options(&options); + + if (options.pass_prompt) { + database_options.password = util::get_password(); + } + + options.conninfo = build_conninfo(database_options); + + if (!options.slim) { + options.middle_database_format = 0; + } + + return options; +} diff -Nru osm2pgsql-1.8.1+ds/src/command-line-parser.hpp osm2pgsql-1.9.0+ds/src/command-line-parser.hpp --- osm2pgsql-1.8.1+ds/src/command-line-parser.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/command-line-parser.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,20 @@ +#ifndef OSM2PGSQL_COMMAND_LINE_PARSER_HPP +#define OSM2PGSQL_COMMAND_LINE_PARSER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "options.hpp" + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) +options_t parse_command_line(int argc, char *argv[]); + +void print_version(); + +#endif // OSM2PGSQL_COMMAND_LINE_PARSER_HPP diff -Nru osm2pgsql-1.8.1+ds/src/db-copy.cpp osm2pgsql-1.9.0+ds/src/db-copy.cpp --- osm2pgsql-1.8.1+ds/src/db-copy.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/db-copy.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -64,7 +64,7 @@ fmt::format_to(std::back_inserter(sql), ") AS t (osm_type, osm_id) WHERE" - " p.{} = t.osm_type AND p.{} = t.osm_id", + " p.{} = t.osm_type::char(1) AND p.{} = t.osm_id", type, column.c_str() + pos + 1); } else { fmt::format_to(std::back_inserter(sql), @@ -131,9 +131,13 @@ // Let commits happen faster by delaying when they actually occur. m_conn->exec("SET synchronous_commit = off"); - // Do not show messages about invalid geometries (they are removed - // by the trigger). - m_conn->exec("SET client_min_messages = WARNING"); + // Disable sequential scan on database tables in the copy threads. + // The copy threads only do COPYs (which are unaffected by this + // setting) and DELETEs which we know benefit from the index. For + // some reason PostgreSQL chooses in some cases not to use that index, + // possibly because the DELETEs get a large list of ids to delete of + // which many are not in the table which confuses the query planner. + m_conn->exec("SET enable_seqscan = off"); bool done = false; while (!done) { @@ -167,7 +171,7 @@ m_conn.reset(); } catch (std::runtime_error const &e) { log_error("DB copy thread failed: {}", e.what()); - exit(2); + std::exit(2); // NOLINT(concurrency-mt-unsafe) } } @@ -184,7 +188,7 @@ start_copy(buffer->target); } - m_conn->copy_send(buffer->buffer, buffer->target->name); + m_conn->copy_send(buffer->buffer, buffer->target->name()); } void db_copy_thread_t::thread_t::start_copy( @@ -192,16 +196,16 @@ { assert(!m_inflight); - auto const qname = qualified_name(target->schema, target->name); + auto const qname = qualified_name(target->schema(), target->name()); fmt::memory_buffer sql; - sql.reserve(qname.size() + target->rows.size() + 20); - if (target->rows.empty()) { + sql.reserve(qname.size() + target->rows().size() + 20); + if (target->rows().empty()) { fmt::format_to(std::back_inserter(sql), FMT_STRING("COPY {} FROM STDIN"), qname); } else { fmt::format_to(std::back_inserter(sql), FMT_STRING("COPY {} ({}) FROM STDIN"), qname, - target->rows); + target->rows()); } sql.push_back('\0'); @@ -213,7 +217,7 @@ void db_copy_thread_t::thread_t::finish_copy() { if (m_inflight) { - m_conn->copy_end(m_inflight->name); + m_conn->copy_end(m_inflight->name()); m_inflight.reset(); } } diff -Nru osm2pgsql-1.8.1+ds/src/db-copy.hpp osm2pgsql-1.9.0+ds/src/db-copy.hpp --- osm2pgsql-1.8.1+ds/src/db-copy.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/db-copy.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,6 +10,7 @@ * For a full list of authors see the git log. */ +#include #include #include #include @@ -26,31 +27,44 @@ /** * Table information necessary for building SQL queries. */ -struct db_target_descr_t +class db_target_descr_t { - /// Schema of the target table (can be empty for default schema) - std::string schema; - /// Name of the target table for the copy operation. - std::string name; - /// Name of id column used when deleting objects. - std::string id; - /// Comma-separated list of rows for copy operation (when empty: all rows) - std::string rows; +public: + db_target_descr_t(std::string schema, std::string name, std::string id, + std::string rows = {}) + : m_schema(std::move(schema)), m_name(std::move(name)), m_id(std::move(id)), + m_rows(std::move(rows)) + { + assert(!m_schema.empty()); + assert(!m_name.empty()); + } + + std::string const &schema() const noexcept { return m_schema; } + std::string const &name() const noexcept { return m_name; } + std::string const &id() const noexcept { return m_id; } + std::string const &rows() const noexcept { return m_rows; } + + void set_rows(std::string rows) { m_rows = std::move(rows); } /** * Check if the buffer would use exactly the same copy operation. */ bool same_copy_target(db_target_descr_t const &other) const noexcept { - return (this == &other) || (schema == other.schema && - name == other.name && rows == other.rows); + return (this == &other) || + (m_schema == other.m_schema && m_name == other.m_name && + m_id == other.m_id && m_rows == other.m_rows); } - db_target_descr_t() = default; - - db_target_descr_t(std::string n, std::string i, std::string r = {}) - : name(std::move(n)), id(std::move(i)), rows(std::move(r)) - {} +private: + /// Schema of the target table. + std::string m_schema; + /// Name of the target table for the copy operation. + std::string m_name; + /// Name of id column used when deleting objects. + std::string m_id; + /// Comma-separated list of rows for copy operation (when empty: all rows) + std::string m_rows; }; /** @@ -198,8 +212,9 @@ void delete_data(pg_conn_t *conn) override { if (m_deleter.has_data()) { - m_deleter.delete_rows(qualified_name(target->schema, target->name), - target->id, conn); + m_deleter.delete_rows( + qualified_name(target->schema(), target->name()), target->id(), + conn); } } diff -Nru osm2pgsql-1.8.1+ds/src/debug-output.cpp osm2pgsql-1.9.0+ds/src/debug-output.cpp --- osm2pgsql-1.8.1+ds/src/debug-output.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/debug-output.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,72 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "debug-output.hpp" + +#include "logging.hpp" + +void write_expire_output_list_to_debug_log( + std::vector const &expire_outputs) +{ + if (!get_logger().debug_enabled()) { + return; + } + + log_debug("ExpireOutputs:"); + std::size_t n = 0; + for (auto const &expire_output : expire_outputs) { + log_debug("- ExpireOutput [{}]", n++); + if (expire_output.minzoom() == expire_output.maxzoom()) { + log_debug(" - zoom: {}", expire_output.maxzoom()); + } else { + log_debug(" - zoom: {}-{}", expire_output.minzoom(), + expire_output.maxzoom()); + } + if (!expire_output.filename().empty()) { + log_debug(" - filename: {}", expire_output.filename()); + } + if (!expire_output.table().empty()) { + log_debug(" - table: {}", qualified_name(expire_output.schema(), + expire_output.table())); + } + } +} + +void write_table_list_to_debug_log(std::vector const &tables) +{ + if (!get_logger().debug_enabled()) { + return; + } + + log_debug("Tables:"); + for (auto const &table : tables) { + log_debug("- Table {}", qualified_name(table.schema(), table.name())); + log_debug(" - columns:"); + for (auto const &column : table) { + log_debug(R"( - "{}" {} ({}) not_null={} create_only={})", + column.name(), column.type_name(), column.sql_type_name(), + column.not_null(), column.create_only()); + for (auto const &ec : column.expire_configs()) { + log_debug(" - expire: [{}]", ec.expire_output); + } + } + log_debug(" - data_tablespace={}", table.data_tablespace()); + log_debug(" - index_tablespace={}", table.index_tablespace()); + log_debug(" - cluster={}", table.cluster_by_geom()); + for (auto const &index : table.indexes()) { + log_debug(" - INDEX USING {}", index.method()); + log_debug(" - column={}", index.columns()); + log_debug(" - expression={}", index.expression()); + log_debug(" - include={}", index.include_columns()); + log_debug(" - tablespace={}", index.tablespace()); + log_debug(" - unique={}", index.is_unique()); + log_debug(" - where={}", index.where_condition()); + } + } +} diff -Nru osm2pgsql-1.8.1+ds/src/debug-output.hpp osm2pgsql-1.9.0+ds/src/debug-output.hpp --- osm2pgsql-1.8.1+ds/src/debug-output.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/debug-output.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,21 @@ +#ifndef OSM2PGSQL_DEBUG_OUTPUT_HPP +#define OSM2PGSQL_DEBUG_OUTPUT_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "expire-output.hpp" +#include "flex-table.hpp" + +void write_expire_output_list_to_debug_log( + std::vector const &expire_outputs); + +void write_table_list_to_debug_log(std::vector const &tables); + +#endif // OSM2PGSQL_DEBUG_OUTPUT_HPP diff -Nru osm2pgsql-1.8.1+ds/src/dependency-manager.cpp osm2pgsql-1.9.0+ds/src/dependency-manager.cpp --- osm2pgsql-1.8.1+ds/src/dependency-manager.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/dependency-manager.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -15,27 +15,89 @@ void full_dependency_manager_t::node_changed(osmid_t id) { - for (auto const way_id : m_object_store->get_ways_by_node(id)) { - way_changed(way_id); - m_ways_pending_tracker.set(way_id); + m_changed_nodes.set(id); +} + +void full_dependency_manager_t::way_changed(osmid_t id) +{ + m_changed_ways.set(id); +} + +void full_dependency_manager_t::relation_changed(osmid_t id) +{ + m_changed_relations.set(id); +} + +void full_dependency_manager_t::after_nodes() +{ + if (m_changed_nodes.empty()) { + return; } - for (auto const rel_id : m_object_store->get_rels_by_node(id)) { - m_rels_pending_tracker.set(rel_id); + m_object_store->get_node_parents(m_changed_nodes, &m_ways_pending_tracker, + &m_rels_pending_tracker); + m_changed_nodes.clear(); +} + +static osmium::index::IdSetSmall +set_diff(osmium::index::IdSetSmall const &set, + osmium::index::IdSetSmall const &to_be_removed) +{ + osmium::index::IdSetSmall new_set; + + for (auto const id : set) { + if (!to_be_removed.get_binary_search(id)) { + new_set.set(id); + } } + + return new_set; } -void full_dependency_manager_t::way_changed(osmid_t id) +void full_dependency_manager_t::after_ways() { - if (m_ways_pending_tracker.get(id)) { + if (!m_changed_ways.empty()) { + if (!m_ways_pending_tracker.empty()) { + // Remove ids from changed ways in the input data from + // m_ways_pending_tracker, because they have already been processed. + m_ways_pending_tracker = + set_diff(m_ways_pending_tracker, m_changed_ways); + + // Add the list of pending way ids to the list of changed ways, + // because we need the parents for them, too. + m_changed_ways.merge_sorted(m_ways_pending_tracker); + } + + m_object_store->get_way_parents(m_changed_ways, + &m_rels_pending_tracker); + + m_changed_ways.clear(); return; } - for (auto const rel_id : m_object_store->get_rels_by_way(id)) { - m_rels_pending_tracker.set(rel_id); + if (!m_ways_pending_tracker.empty()) { + m_object_store->get_way_parents(m_ways_pending_tracker, + &m_rels_pending_tracker); } } +void full_dependency_manager_t::after_relations() +{ + // Remove ids from changed relations in the input data from + // m_rels_pending_tracker, because they have already been processed. + m_rels_pending_tracker = + set_diff(m_rels_pending_tracker, m_changed_relations); + + m_changed_relations.clear(); +} + +void full_dependency_manager_t::mark_parent_relations_as_pending( + osmium::index::IdSetSmall const &way_ids) +{ + assert(m_rels_pending_tracker.empty()); + m_object_store->get_way_parents(way_ids, &m_rels_pending_tracker); +} + bool full_dependency_manager_t::has_pending() const noexcept { return !m_ways_pending_tracker.empty() || !m_rels_pending_tracker.empty(); diff -Nru osm2pgsql-1.8.1+ds/src/dependency-manager.hpp osm2pgsql-1.9.0+ds/src/dependency-manager.hpp --- osm2pgsql-1.8.1+ds/src/dependency-manager.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/dependency-manager.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -52,6 +52,17 @@ */ virtual void way_changed(osmid_t) {} + virtual void relation_changed(osmid_t) {} + + virtual void after_nodes() {} + virtual void after_ways() {} + virtual void after_relations() {} + + virtual void mark_parent_relations_as_pending( + osmium::index::IdSetSmall const & /*way_ids*/) + { + } + /// Are there pending objects that need to be processed? virtual bool has_pending() const noexcept { return false; } @@ -94,6 +105,14 @@ void node_changed(osmid_t id) override; void way_changed(osmid_t id) override; + void relation_changed(osmid_t id) override; + + void after_nodes() override; + void after_ways() override; + void after_relations() override; + + void mark_parent_relations_as_pending( + osmium::index::IdSetSmall const &ids) override; bool has_pending() const noexcept override; @@ -112,6 +131,32 @@ std::shared_ptr m_object_store; + /** + * In append mode all new and changed nodes will be added to this. After + * all nodes are read this is used to figure out which parent ways and + * relations reference these nodes. Deleted nodes are not stored in here, + * because all ways and relations that referenced deleted nodes must be in + * the change file, too, and so we don't have to find out which ones they + * are. + */ + osmium::index::IdSetSmall m_changed_nodes; + + /** + * In append mode all new and changed ways will be added to this. After + * all ways are read this is used to figure out which parent relations + * reference these ways. Deleted ways are not stored in here, because all + * relations that referenced deleted ways must be in the change file, too, + * and so we don't have to find out which ones they are. + */ + osmium::index::IdSetSmall m_changed_ways; + + /** + * In append mode all new and changed relations will be added to this. + * This is then used to remove already processed relations from the + * pending list. + */ + osmium::index::IdSetSmall m_changed_relations; + osmium::index::IdSetSmall m_ways_pending_tracker; osmium::index::IdSetSmall m_rels_pending_tracker; }; diff -Nru osm2pgsql-1.8.1+ds/src/expire-config.hpp osm2pgsql-1.9.0+ds/src/expire-config.hpp --- osm2pgsql-1.8.1+ds/src/expire-config.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/expire-config.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -14,7 +14,7 @@ enum class expire_mode { - full_area, // Expire all tiles covered by polygon. + full_area, // Expire all tiles covered by polygon. boundary_only, // Expire only tiles covered by polygon boundary. hybrid // "full_area" or "boundary_only" mode depending on full_area_limit. }; @@ -24,6 +24,12 @@ */ struct expire_config_t { + /** + * The id of the expire output to which expired tiles are written. + * Only used in the flex output. + */ + std::size_t expire_output = 0; + /// Buffer around expired feature as fraction of the tile size. double buffer = 0.1; diff -Nru osm2pgsql-1.8.1+ds/src/expire-output.cpp osm2pgsql-1.9.0+ds/src/expire-output.cpp --- osm2pgsql-1.8.1+ds/src/expire-output.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/expire-output.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,85 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "expire-output.hpp" + +#include "format.hpp" +#include "logging.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +#include + +std::size_t expire_output_t::output(quadkey_list_t const &tile_list, + std::string const &conninfo) const +{ + std::size_t num = 0; + if (!m_filename.empty()) { + num = output_tiles_to_file(tile_list); + } + if (!m_table.empty()) { + num = output_tiles_to_table(tile_list, conninfo); + } + return num; +} + +std::size_t expire_output_t::output_tiles_to_file( + quadkey_list_t const &tiles_at_maxzoom) const +{ + FILE *outfile = std::fopen(m_filename.data(), "a"); + if (outfile == nullptr) { + std::system_error const error{errno, std::generic_category()}; + log_warn("Failed to open expired tiles file ({}). Tile expiry " + "list will not be written!", + error.code().message()); + return 0; + } + + auto const count = for_each_tile( + tiles_at_maxzoom, m_minzoom, m_maxzoom, [&](tile_t const &tile) { + fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y()); + }); + + (void)std::fclose(outfile); + + return count; +} + +std::size_t +expire_output_t::output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, + std::string const &conninfo) const +{ + auto const qn = qualified_name(m_schema, m_table); + + pg_conn_t connection{conninfo}; + + connection.exec("PREPARE insert_tiles(int4, int4, int4) AS" + " INSERT INTO {} (zoom, x, y) VALUES ($1, $2, $3)" + " ON CONFLICT DO NOTHING", + qn); + + auto const count = for_each_tile( + tiles_at_maxzoom, m_minzoom, m_maxzoom, [&](tile_t const &tile) { + connection.exec_prepared("insert_tiles", tile.zoom(), tile.x(), + tile.y()); + }); + + return count; +} + +void expire_output_t::create_output_table(pg_conn_t const &connection) const +{ + auto const qn = qualified_name(m_schema, m_table); + connection.exec("CREATE TABLE IF NOT EXISTS {} (" + " zoom int4 NOT NULL," + " x int4 NOT NULL," + " y int4 NOT NULL," + " PRIMARY KEY (zoom, x, y))", + qn); +} diff -Nru osm2pgsql-1.8.1+ds/src/expire-output.hpp osm2pgsql-1.9.0+ds/src/expire-output.hpp --- osm2pgsql-1.8.1+ds/src/expire-output.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/expire-output.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,97 @@ +#ifndef OSM2PGSQL_EXPIRE_OUTPUT_HPP +#define OSM2PGSQL_EXPIRE_OUTPUT_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "tile.hpp" + +#include +#include +#include +#include + +class pg_conn_t; + +/** + * Output for tile expiry. + */ +class expire_output_t +{ +public: + expire_output_t() = default; + + std::string filename() const noexcept { return m_filename; } + + void set_filename(std::string filename) + { + m_filename = std::move(filename); + } + + std::string schema() const noexcept { return m_schema; } + + std::string table() const noexcept { return m_table; } + + void set_schema_and_table(std::string schema, std::string table) + { + assert(!schema.empty()); + m_schema = std::move(schema); + m_table = std::move(table); + } + + uint32_t minzoom() const noexcept { return m_minzoom; } + void set_minzoom(uint32_t minzoom) noexcept { m_minzoom = minzoom; } + + uint32_t maxzoom() const noexcept { return m_maxzoom; } + void set_maxzoom(uint32_t maxzoom) noexcept { m_maxzoom = maxzoom; } + + std::size_t output(quadkey_list_t const &tile_list, + std::string const &conninfo) const; + + /** + * Write the list of tiles to a file. + * + * \param tiles_at_maxzoom The list of tiles at maximum zoom level + */ + std::size_t + output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom) const; + + /** + * Write the list of tiles to a database table. + * + * \param tiles_at_maxzoom The list of tiles at maximum zoom level + * \param conninfo database connection info + */ + std::size_t output_tiles_to_table(quadkey_list_t const &tiles_at_maxzoom, + std::string const &conninfo) const; + + /** + * Create table for tiles. + */ + void create_output_table(pg_conn_t const &connection) const; + +private: + /// The filename (if any) for output + std::string m_filename; + + /// The schema for output + std::string m_schema; + + /// The table (if any) for output + std::string m_table; + + /// Minimum zoom level for output + uint32_t m_minzoom = 0; + + /// Zoom level we capture tiles on + uint32_t m_maxzoom = 0; + +}; // class expire_output_t + +#endif // OSM2PGSQL_EXPIRE_OUTPUT_HPP diff -Nru osm2pgsql-1.8.1+ds/src/expire-tiles.cpp osm2pgsql-1.9.0+ds/src/expire-tiles.cpp --- osm2pgsql-1.8.1+ds/src/expire-tiles.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/expire-tiles.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -27,7 +27,6 @@ #include "expire-tiles.hpp" #include "format.hpp" #include "geom-functions.hpp" -#include "logging.hpp" #include "options.hpp" #include "reprojection.hpp" #include "table.hpp" @@ -292,28 +291,6 @@ } } -std::size_t output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, - std::string_view filename) -{ - FILE *outfile = std::fopen(filename.data(), "a"); - if (outfile == nullptr) { - log_warn("Failed to open expired tiles file ({}). Tile expiry " - "list will not be written!", - std::strerror(errno)); - return 0; - } - - auto const count = for_each_tile( - tiles_at_maxzoom, minzoom, maxzoom, [&](tile_t const &tile) { - fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y()); - }); - - (void)std::fclose(outfile); - - return count; -} - int expire_from_result(expire_tiles *expire, pg_result_t const &result, expire_config_t const &expire_config) { diff -Nru osm2pgsql-1.8.1+ds/src/expire-tiles.hpp osm2pgsql-1.9.0+ds/src/expire-tiles.hpp --- osm2pgsql-1.8.1+ds/src/expire-tiles.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/expire-tiles.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,6 +10,7 @@ * For a full list of authors see the git log. */ +#include #include #include #include @@ -31,6 +32,8 @@ public: expire_tiles(uint32_t max_zoom, std::shared_ptr projection); + bool empty() const noexcept { return m_dirty_tiles.empty(); } + bool enabled() const noexcept { return m_maxzoom != 0; } void from_polygon_boundary(geom::polygon_t const &geom, @@ -85,7 +88,6 @@ void merge_and_destroy(expire_tiles *other); private: - /** * Converts from target coordinates to tile coordinates. */ @@ -133,67 +135,4 @@ int expire_from_result(expire_tiles *expire, pg_result_t const &result, expire_config_t const &expire_config); -/** - * Iterate over tiles and call output function for each tile on all requested - * zoom levels. - * - * \tparam OUTPUT Class with operator() taking a tile_t argument - * - * \param tiles_at_maxzoom The list of tiles at maximum zoom level - * \param minzoom Minimum zoom level - * \param maxzoom Maximum zoom level - * \param output Output function - */ -template -std::size_t for_each_tile(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, OUTPUT &&output) -{ - assert(minzoom <= maxzoom); - - if (minzoom == maxzoom) { - for (auto const quadkey : tiles_at_maxzoom) { - std::forward(output)( - tile_t::from_quadkey(quadkey, maxzoom)); - } - return tiles_at_maxzoom.size(); - } - - /** - * Loop over all requested zoom levels (from maximum down to the minimum - * zoom level). - */ - quadkey_t last_quadkey{}; - std::size_t count = 0; - for (auto const quadkey : tiles_at_maxzoom) { - for (uint32_t dz = 0; dz <= maxzoom - minzoom; ++dz) { - auto const qt_current = quadkey.down(dz); - /** - * If dz > 0, there are probably multiple elements whose quadkey - * is equal because they are all sub-tiles of the same tile at the - * current zoom level. We skip all of them after we have written - * the first sibling. - */ - if (qt_current != last_quadkey.down(dz)) { - std::forward(output)( - tile_t::from_quadkey(qt_current, maxzoom - dz)); - ++count; - } - } - last_quadkey = quadkey; - } - return count; -} - -/** - * Write the list of tiles to a file. - * - * \param tiles_at_maxzoom The list of tiles at maximum zoom level - * \param minzoom Minimum zoom level - * \param maxzoom Maximum zoom level - * \param filename Name of the file - */ -std::size_t output_tiles_to_file(quadkey_list_t const &tiles_at_maxzoom, - uint32_t minzoom, uint32_t maxzoom, - std::string_view filename); - #endif // OSM2PGSQL_EXPIRE_TILES_HPP diff -Nru osm2pgsql-1.8.1+ds/src/flex-index.hpp osm2pgsql-1.9.0+ds/src/flex-index.hpp --- osm2pgsql-1.8.1+ds/src/flex-index.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-index.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -11,6 +11,7 @@ */ #include +#include #include #include #include diff -Nru osm2pgsql-1.8.1+ds/src/flex-lua-expire-output.cpp osm2pgsql-1.9.0+ds/src/flex-lua-expire-output.cpp --- osm2pgsql-1.8.1+ds/src/flex-lua-expire-output.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-lua-expire-output.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,92 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "flex-lua-expire-output.hpp" + +#include "expire-output.hpp" +#include "format.hpp" +#include "lua-utils.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include + +static expire_output_t & +create_expire_output(lua_State *lua_state, std::string const &default_schema, + std::vector *expire_outputs) +{ + auto &new_expire_output = expire_outputs->emplace_back(); + + // optional "filename" field + auto const *filename = luaX_get_table_string(lua_state, "filename", -1, + "The expire output", ""); + new_expire_output.set_filename(filename); + lua_pop(lua_state, 1); // "filename" + + // optional "schema" and "table" fields + auto const *schema = luaX_get_table_string( + lua_state, "schema", -1, "The expire output", default_schema.c_str()); + check_identifier(schema, "schema field"); + auto const *table = + luaX_get_table_string(lua_state, "table", -2, "The expire output", ""); + check_identifier(table, "table field"); + new_expire_output.set_schema_and_table(schema, table); + lua_pop(lua_state, 2); // "schema" and "table" + + if (new_expire_output.filename().empty() && + new_expire_output.table().empty()) { + throw std::runtime_error{ + "Must set 'filename' and/or 'table' on expire output."}; + } + + // required "maxzoom" field + auto value = luaX_get_table_optional_uint32( + lua_state, "maxzoom", -1, "The 'maxzoom' field in a expire output"); + if (value >= 1 && value <= 20) { + new_expire_output.set_minzoom(value); + new_expire_output.set_maxzoom(value); + } else { + throw std::runtime_error{ + "Value of 'maxzoom' field must be between 1 and 20."}; + } + lua_pop(lua_state, 1); // "maxzoom" + + // optional "minzoom" field + value = luaX_get_table_optional_uint32( + lua_state, "minzoom", -1, "The 'minzoom' field in a expire output"); + if (value >= 1 && value <= new_expire_output.maxzoom()) { + new_expire_output.set_minzoom(value); + } else if (value != 0) { + throw std::runtime_error{ + "Value of 'minzoom' field must be between 1 and 'maxzoom'."}; + } + lua_pop(lua_state, 1); // "minzoom" + + return new_expire_output; +} + +int setup_flex_expire_output(lua_State *lua_state, + std::string const &default_schema, + std::vector *expire_outputs) +{ + if (lua_type(lua_state, 1) != LUA_TTABLE) { + throw std::runtime_error{ + "Argument #1 to 'define_expire_output' must be a Lua table."}; + } + + create_expire_output(lua_state, default_schema, expire_outputs); + + void *ptr = lua_newuserdata(lua_state, sizeof(std::size_t)); + auto *num = new (ptr) std::size_t{}; + *num = expire_outputs->size() - 1; + luaL_getmetatable(lua_state, osm2pgsql_expire_output_name); + lua_setmetatable(lua_state, -2); + + return 1; +} diff -Nru osm2pgsql-1.8.1+ds/src/flex-lua-expire-output.hpp osm2pgsql-1.9.0+ds/src/flex-lua-expire-output.hpp --- osm2pgsql-1.8.1+ds/src/flex-lua-expire-output.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-lua-expire-output.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,26 @@ +#ifndef OSM2PGSQL_FLEX_LUA_EXPIRE_OUTPUT_HPP +#define OSM2PGSQL_FLEX_LUA_EXPIRE_OUTPUT_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "expire-output.hpp" + +#include + +struct lua_State; + +static char const *const osm2pgsql_expire_output_name = + "osm2pgsql.ExpireOutput"; + +int setup_flex_expire_output(lua_State *lua_state, + std::string const &default_schema, + std::vector *expire_outputs); + +#endif // OSM2PGSQL_FLEX_LUA_EXPIRE_OUTPUT_HPP diff -Nru osm2pgsql-1.8.1+ds/src/flex-lua-geom.cpp osm2pgsql-1.9.0+ds/src/flex-lua-geom.cpp --- osm2pgsql-1.8.1+ds/src/flex-lua-geom.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-lua-geom.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -12,11 +12,7 @@ #include "geom-pole-of-inaccessibility.hpp" #include "lua-utils.hpp" -extern "C" -{ -#include -#include -} +#include static char const *const osm2pgsql_geometry_class = "osm2pgsql.Geometry"; @@ -70,6 +66,24 @@ return 1; } +static int geom_spherical_area(lua_State *lua_state) +{ + auto const *const input_geometry = unpack_geometry(lua_state); + + if (input_geometry->srid() != 4326) { + throw std::runtime_error{"Can only calculate spherical area for " + "geometries in WGS84 (4326) coordinates."}; + } + + try { + lua_pushnumber(lua_state, geom::spherical_area(*input_geometry)); + } catch (...) { + return luaL_error(lua_state, "Unknown error in 'spherical_area()'.\n"); + } + + return 1; +} + static int geom_length(lua_State *lua_state) { auto const *const input_geometry = unpack_geometry(lua_state); @@ -290,6 +304,7 @@ geom_pole_of_inaccessibility); luaX_add_table_func(lua_state, "segmentize", geom_segmentize); luaX_add_table_func(lua_state, "simplify", geom_simplify); + luaX_add_table_func(lua_state, "spherical_area", geom_spherical_area); luaX_add_table_func(lua_state, "srid", geom_srid); luaX_add_table_func(lua_state, "transform", geom_transform); diff -Nru osm2pgsql-1.8.1+ds/src/flex-lua-table.cpp osm2pgsql-1.9.0+ds/src/flex-lua-table.cpp --- osm2pgsql-1.8.1+ds/src/flex-lua-table.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-lua-table.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -7,10 +7,13 @@ * For a full list of authors see the git log. */ -#include "flex-lua-index.hpp" #include "flex-lua-table.hpp" -#include "lua-utils.hpp" + +#include "expire-output.hpp" +#include "flex-lua-expire-output.hpp" +#include "flex-lua-index.hpp" #include "flex-table.hpp" +#include "lua-utils.hpp" #include "pgsql-capabilities.hpp" #include @@ -26,6 +29,7 @@ } static flex_table_t &create_flex_table(lua_State *lua_state, + std::string const &default_schema, std::vector *tables) { std::string const table_name = @@ -37,7 +41,7 @@ throw fmt_error("Table with name '{}' already exists.", table_name); } - auto &new_table = tables->emplace_back(table_name); + auto &new_table = tables->emplace_back(default_schema, table_name); lua_pop(lua_state, 1); // "name" @@ -46,11 +50,7 @@ if (lua_isstring(lua_state, -1)) { std::string const schema = lua_tostring(lua_state, -1); check_identifier(schema, "schema field"); - if (!has_schema(schema)) { - throw fmt_error("Schema '{0}' not available." - " Use 'CREATE SCHEMA \"{0}\";' to create it.", - schema); - } + check_schema(schema); new_table.set_schema(schema); } lua_pop(lua_state, 1); @@ -100,6 +100,19 @@ return new_table; } +static void parse_create_index(lua_State *lua_state, flex_table_t *table) +{ + std::string const create_index = luaX_get_table_string( + lua_state, "create_index", -1, "The ids field", "auto"); + lua_pop(lua_state, 1); // "create_index" + if (create_index == "always") { + table->set_always_build_id_index(); + } else if (create_index != "auto") { + throw fmt_error("Unknown value '{}' for 'create_index' field of ids", + create_index); + } +} + static void setup_flex_table_id_columns(lua_State *lua_state, flex_table_t *table) { @@ -120,15 +133,15 @@ lua_pop(lua_state, 1); // "type" if (type == "node") { - table->set_id_type(osmium::item_type::node); + table->set_id_type(flex_table_index_type::node); } else if (type == "way") { - table->set_id_type(osmium::item_type::way); + table->set_id_type(flex_table_index_type::way); } else if (type == "relation") { - table->set_id_type(osmium::item_type::relation); + table->set_id_type(flex_table_index_type::relation); } else if (type == "area") { - table->set_id_type(osmium::item_type::area); + table->set_id_type(flex_table_index_type::area); } else if (type == "any") { - table->set_id_type(osmium::item_type::undefined); + table->set_id_type(flex_table_index_type::any_object); lua_getfield(lua_state, -1, "type_column"); if (lua_isstring(lua_state, -1)) { std::string const column_name = @@ -140,6 +153,13 @@ throw std::runtime_error{"type_column must be a string or nil."}; } lua_pop(lua_state, 1); // "type_column" + } else if (type == "tile") { + table->set_id_type(flex_table_index_type::tile); + parse_create_index(lua_state, table); + table->add_column("x", "int", "int").set_not_null(); + table->add_column("y", "int", "int").set_not_null(); + lua_pop(lua_state, 1); // "ids" + return; } else { throw fmt_error("Unknown ids type: {}.", type); } @@ -149,22 +169,137 @@ lua_pop(lua_state, 1); // "id_column" check_identifier(name, "column names"); - std::string const create_index = luaX_get_table_string( - lua_state, "create_index", -1, "The ids field", "auto"); - lua_pop(lua_state, 1); // "create_index" - if (create_index == "always") { - table->set_always_build_id_index(); - } else if (create_index != "auto") { - throw fmt_error("Unknown value '{}' for 'create_index' field of ids", - create_index); - } + parse_create_index(lua_state, table); auto &column = table->add_column(name, "id_num", ""); column.set_not_null(); lua_pop(lua_state, 1); // "ids" } -static void setup_flex_table_columns(lua_State *lua_state, flex_table_t *table) +static std::size_t idx_from_userdata(lua_State *lua_state, int idx, + std::size_t expire_outputs_size) +{ + void const *const user_data = lua_touserdata(lua_state, idx); + + if (user_data == nullptr || !lua_getmetatable(lua_state, idx)) { + throw std::runtime_error{"Expire output must be of type ExpireOutput."}; + } + + luaL_getmetatable(lua_state, osm2pgsql_expire_output_name); + if (!lua_rawequal(lua_state, -1, -2)) { + throw std::runtime_error{"Expire output must be of type ExpireOutput."}; + } + lua_pop(lua_state, 2); // remove the two metatables + + auto const eo = *static_cast(user_data); + if (eo >= expire_outputs_size) { + throw std::runtime_error{"Internal error in expire output code."}; + } + return eo; +} + +static void parse_and_set_expire_options(lua_State *lua_state, + flex_table_column_t *column, + std::size_t expire_outputs_size, + bool append_mode) +{ + auto const type = lua_type(lua_state, -1); + + if (type == LUA_TNIL) { + return; + } + + if (!column->is_geometry_column() || column->srid() != 3857) { + throw std::runtime_error{"Expire only allowed for geometry" + " columns in Web Mercator projection."}; + } + + if (type == LUA_TUSERDATA) { + auto const eo = idx_from_userdata(lua_state, -1, expire_outputs_size); + // Actually add the expire only if we are in append mode. + if (append_mode) { + expire_config_t const config{eo}; + column->add_expire(config); + } + return; + } + + if (type != LUA_TTABLE) { + throw std::runtime_error{"Expire field must be a Lua array table"}; + } + + if (luaX_is_empty_table(lua_state)) { + return; + } + + if (!luaX_is_array(lua_state)) { + throw std::runtime_error{"Expire field must be a Lua array table"}; + } + + luaX_for_each(lua_state, [&]() { + if (!lua_istable(lua_state, -1) || luaX_is_array(lua_state)) { + throw std::runtime_error{"Expire config must be a Lua table"}; + } + + lua_getfield(lua_state, -1, "output"); + auto const eo = idx_from_userdata(lua_state, -1, expire_outputs_size); + lua_pop(lua_state, 1); // "output" + + expire_config_t config{eo}; + + std::string mode; + lua_getfield(lua_state, -1, "mode"); + if (lua_isstring(lua_state, -1)) { + mode = lua_tostring(lua_state, -1); + } else if (!lua_isnil(lua_state, -1)) { + throw std::runtime_error{ + "Optional expire field 'mode' must contain a string."}; + } + lua_pop(lua_state, 1); // "mode" + + if (mode.empty() || mode == "full-area") { + config.mode = expire_mode::full_area; + } else if (mode == "boundary-only") { + config.mode = expire_mode::boundary_only; + } else if (mode == "hybrid") { + config.mode = expire_mode::hybrid; + } else { + throw fmt_error("Unknown expire mode '{}'.", mode); + } + + lua_getfield(lua_state, -1, "full_area_limit"); + if (lua_isnumber(lua_state, -1)) { + if (config.mode != expire_mode::hybrid) { + log_warn("Ignoring 'full_area_limit' setting in expire config," + " because 'mode' is not set to 'hybrid'."); + } + config.full_area_limit = lua_tonumber(lua_state, -1); + } else if (!lua_isnil(lua_state, -1)) { + throw std::runtime_error{"Optional expire field 'full_area_limit' " + "must contain a number."}; + } + lua_pop(lua_state, 1); // "full_area_limit" + + lua_getfield(lua_state, -1, "buffer"); + if (lua_isnumber(lua_state, -1)) { + config.buffer = lua_tonumber(lua_state, -1); + } else if (!lua_isnil(lua_state, -1)) { + throw std::runtime_error{ + "Optional expire field 'buffer' must contain a number."}; + } + lua_pop(lua_state, 1); // "buffer" + + // Actually add the expire only if we are in append mode. + if (append_mode) { + column->add_expire(config); + } + }); +} + +static void +setup_flex_table_columns(lua_State *lua_state, flex_table_t *table, + std::vector *expire_outputs, + bool append_mode) { assert(lua_state); assert(table); @@ -216,6 +351,11 @@ } lua_pop(lua_state, 1); // "projection" + lua_getfield(lua_state, -1, "expire"); + parse_and_set_expire_options(lua_state, &column, expire_outputs->size(), + append_mode); + lua_pop(lua_state, 1); // "expire" + ++num_columns; }); @@ -271,16 +411,19 @@ } int setup_flex_table(lua_State *lua_state, std::vector *tables, - bool updatable) + std::vector *expire_outputs, + std::string const &default_schema, bool updatable, + bool append_mode) { if (lua_type(lua_state, 1) != LUA_TTABLE) { throw std::runtime_error{ "Argument #1 to 'define_table' must be a table."}; } - auto &new_table = create_flex_table(lua_state, tables); + auto &new_table = create_flex_table(lua_state, default_schema, tables); setup_flex_table_id_columns(lua_state, &new_table); - setup_flex_table_columns(lua_state, &new_table); + setup_flex_table_columns(lua_state, &new_table, expire_outputs, + append_mode); setup_flex_table_indexes(lua_state, &new_table, updatable); void *ptr = lua_newuserdata(lua_state, sizeof(std::size_t)); diff -Nru osm2pgsql-1.8.1+ds/src/flex-lua-table.hpp osm2pgsql-1.9.0+ds/src/flex-lua-table.hpp --- osm2pgsql-1.8.1+ds/src/flex-lua-table.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-lua-table.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,14 +10,18 @@ * For a full list of authors see the git log. */ +#include #include +class expire_output_t; class flex_table_t; struct lua_State; static char const *const osm2pgsql_table_name = "osm2pgsql.Table"; int setup_flex_table(lua_State *lua_state, std::vector *tables, - bool updatable); + std::vector *expire_outputs, + std::string const &default_schema, bool updatable, + bool append_mode); #endif // OSM2PGSQL_FLEX_LUA_TABLE_HPP diff -Nru osm2pgsql-1.8.1+ds/src/flex-table-column.cpp osm2pgsql-1.9.0+ds/src/flex-table-column.cpp --- osm2pgsql-1.8.1+ds/src/flex-table-column.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-table-column.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -191,3 +191,21 @@ return fmt::format(R"("{}" {} {})", m_name, sql_type_name(), sql_modifiers()); } + +void flex_table_column_t::add_expire(expire_config_t const &config) +{ + assert(is_geometry_column()); + assert(srid() == 3857); + m_expires.push_back(config); +} + +void flex_table_column_t::do_expire(geom::geometry_t const &geom, + std::vector *expire) const +{ + assert(expire); + for (auto const &expire_config : m_expires) { + assert(expire_config.expire_output < expire->size()); + (*expire)[expire_config.expire_output].from_geometry(geom, + expire_config); + } +} diff -Nru osm2pgsql-1.8.1+ds/src/flex-table-column.hpp osm2pgsql-1.9.0+ds/src/flex-table-column.hpp --- osm2pgsql-1.8.1+ds/src/flex-table-column.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-table-column.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,9 +10,14 @@ * For a full list of authors see the git log. */ +#include "expire-config.hpp" +#include "expire-tiles.hpp" +#include "geom.hpp" + #include #include #include +#include enum class table_column_type : uint8_t { @@ -118,7 +123,21 @@ int srid() const noexcept { return m_srid; } + void add_expire(expire_config_t const &config); + + bool has_expire() const noexcept { return !m_expires.empty(); } + + std::vector const &expire_configs() const noexcept + { + return m_expires; + } + + void do_expire(geom::geometry_t const &geom, + std::vector *expire) const; + private: + std::vector m_expires; + /// The name of the database table column. std::string m_name; diff -Nru osm2pgsql-1.8.1+ds/src/flex-table.cpp osm2pgsql-1.9.0+ds/src/flex-table.cpp --- osm2pgsql-1.8.1+ds/src/flex-table.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-table.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -77,30 +77,35 @@ bool flex_table_t::matches_type(osmium::item_type type) const noexcept { // This table takes any type -> okay - if (m_id_type == osmium::item_type::undefined) { + if (m_id_type == flex_table_index_type::any_object) { return true; } // Type and table type match -> okay - if (type == m_id_type) { + if ((type == osmium::item_type::node && + m_id_type == flex_table_index_type::node) || + (type == osmium::item_type::way && + m_id_type == flex_table_index_type::way) || + (type == osmium::item_type::relation && + m_id_type == flex_table_index_type::relation)) { return true; } // Relations can be written as linestrings into way tables -> okay if (type == osmium::item_type::relation && - m_id_type == osmium::item_type::way) { + m_id_type == flex_table_index_type::way) { return true; } // Area tables can take ways or relations, but not nodes - return m_id_type == osmium::item_type::area && + return m_id_type == flex_table_index_type::area && type != osmium::item_type::node; } /// Map way/node/relation ID to id value used in database table column osmid_t flex_table_t::map_id(osmium::item_type type, osmid_t id) const noexcept { - if (m_id_type == osmium::item_type::undefined) { + if (m_id_type == flex_table_index_type::any_object) { if (has_multicolumn_id_index()) { return id; } @@ -117,7 +122,7 @@ } } - if (m_id_type != osmium::item_type::relation && + if (m_id_type != flex_table_index_type::relation && type == osmium::item_type::relation) { return -id; } @@ -149,17 +154,27 @@ std::string flex_table_t::build_sql_prepare_get_wkb() const { + util::string_joiner_t joiner{',', '"'}; + for (auto const &column : m_columns) { + if (!column.expire_configs().empty()) { + joiner.add(column.name()); + } + } + + assert(!joiner.empty()); + + std::string const columns = joiner(); + if (has_multicolumn_id_index()) { return fmt::format( - "PREPARE get_wkb(char(1), bigint) AS" - " SELECT \"{}\" FROM {} WHERE \"{}\" = $1 AND \"{}\" = $2", - geom_column().name(), full_name(), m_columns[0].name(), - m_columns[1].name()); + R"(PREPARE get_wkb(char(1), bigint) AS)" + R"( SELECT {} FROM {} WHERE "{}" = $1 AND "{}" = $2)", + columns, full_name(), m_columns[0].name(), m_columns[1].name()); } - return fmt::format("PREPARE get_wkb(bigint) AS" - " SELECT \"{}\" FROM {} WHERE \"{}\" = $1", - geom_column().name(), full_name(), id_column_names()); + return fmt::format(R"(PREPARE get_wkb(bigint) AS)" + R"( SELECT {} FROM {} WHERE "{}" = $1)", + columns, full_name(), id_column_names()); } std::string @@ -220,6 +235,12 @@ return m_indexes.emplace_back(std::move(method)); } +bool flex_table_t::has_columns_with_expire() const noexcept +{ + return std::any_of(m_columns.cbegin(), m_columns.cend(), + [](auto const &column) { return column.has_expire(); }); +} + void table_connection_t::connect(std::string const &conninfo) { assert(!m_db_connection); @@ -256,8 +277,6 @@ { assert(m_db_connection); - m_db_connection->exec("SET client_min_messages = WARNING"); - if (!append) { m_db_connection->exec("DROP TABLE IF EXISTS {} CASCADE", table().full_name()); @@ -265,7 +284,6 @@ // These _tmp tables can be left behind if we run out of disk space. m_db_connection->exec("DROP TABLE IF EXISTS {}", table().full_tmp_name()); - m_db_connection->exec("RESET client_min_messages"); if (!append) { m_db_connection->exec(table().build_sql_create_table( @@ -298,10 +316,6 @@ log_info("Clustering table '{}' by geometry...", table().name()); - // Notices about invalid geometries are expected and can be ignored - // because they say nothing about the validity of the geometry in OSM. - m_db_connection->exec("SET client_min_messages = WARNING"); - m_db_connection->exec(table().build_sql_create_table( flex_table_t::table_type::permanent, table().full_tmp_name())); @@ -359,8 +373,8 @@ } } - if (table().always_build_id_index() || - (updateable && table().has_id_column())) { + if ((table().always_build_id_index() || updateable) && + table().has_id_column()) { create_id_index(); } @@ -373,7 +387,7 @@ void table_connection_t::prepare() { assert(m_db_connection); - if (table().has_id_column() && table().has_geom_column()) { + if (table().has_id_column() && table().has_columns_with_expire()) { m_db_connection->exec(table().build_sql_prepare_get_wkb()); } } @@ -394,8 +408,8 @@ } } -pg_result_t table_connection_t::get_geom_by_id(osmium::item_type type, - osmid_t id) const +pg_result_t table_connection_t::get_geoms_by_id(osmium::item_type type, + osmid_t id) const { assert(table().has_geom_column()); assert(m_db_connection); diff -Nru osm2pgsql-1.8.1+ds/src/flex-table.hpp osm2pgsql-1.9.0+ds/src/flex-table.hpp --- osm2pgsql-1.8.1+ds/src/flex-table.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-table.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -16,6 +16,7 @@ #include "pgsql.hpp" #include "reprojection.hpp" #include "thread-pool.hpp" +#include "util.hpp" #include @@ -27,6 +28,21 @@ #include /** + * This defines the type of "primary key" for the tables generated in the flex + * output. This is not a real primary key, because the values are not + * necessarily unique. + */ +enum class flex_table_index_type { + no_index, + node, // index by node id + way, // index by way id + relation, // index by relation id + area, // index by way (positive) or relation (negative) id + any_object, // any OSM object, two columns for type and id + tile // index by tile with x and y columns (used for generalized data) +}; + +/** * An output table (in the SQL sense) for the flex backend. */ class flex_table_t @@ -43,7 +59,10 @@ permanent }; - explicit flex_table_t(std::string name) : m_name(std::move(name)) {} + flex_table_t(std::string schema, std::string name) + : m_schema(std::move(schema)), m_name(std::move(name)) + { + } std::string const &name() const noexcept { return m_name; } @@ -84,9 +103,9 @@ m_index_tablespace = std::move(tablespace); } - osmium::item_type id_type() const noexcept { return m_id_type; } + flex_table_index_type id_type() const noexcept { return m_id_type; } - void set_id_type(osmium::item_type type) noexcept { m_id_type = type; } + void set_id_type(flex_table_index_type type) noexcept { m_id_type = type; } bool has_id_column() const noexcept; @@ -102,6 +121,11 @@ return m_columns.end(); } + flex_table_column_t *find_column_by_name(std::string const &name) + { + return util::find_by_name(m_columns, name); + } + bool has_geom_column() const noexcept { return m_geom_column != std::numeric_limits::max(); @@ -114,6 +138,12 @@ return m_columns[m_geom_column]; } + flex_table_column_t &geom_column() noexcept + { + assert(has_geom_column()); + return m_columns[m_geom_column]; + } + int srid() const noexcept { return has_geom_column() ? geom_column().srid() : 4326; @@ -165,13 +195,15 @@ return m_always_build_id_index; } + bool has_columns_with_expire() const noexcept; + private: + /// The schema this table is in + std::string m_schema; + /// The name of the table std::string m_name; - /// The schema this table is in - std::string m_schema{"public"}; - /// The table space used for this table (empty for default tablespace) std::string m_data_tablespace; @@ -199,10 +231,9 @@ std::size_t m_geom_column = std::numeric_limits::max(); /** - * Type of Id stored in this table (node, way, relation, area, or - * undefined for any type). + * Type of id stored in this table. */ - osmium::item_type m_id_type = osmium::item_type::undefined; + flex_table_index_type m_id_type = flex_table_index_type::no_index; /// Cluster the table by geometry. bool m_cluster_by_geom = true; @@ -222,11 +253,10 @@ std::shared_ptr const ©_thread) : m_proj(reprojection::create_projection(table->srid())), m_table(table), m_target(std::make_shared( - table->name(), table->id_column_names(), + table->schema(), table->name(), table->id_column_names(), table->build_sql_column_list())), m_copy_mgr(copy_thread), m_db_connection(nullptr) { - m_target->schema = table->schema(); } void connect(std::string const &conninfo); @@ -245,7 +275,11 @@ void create_id_index(); - pg_result_t get_geom_by_id(osmium::item_type type, osmid_t id) const; + /** + * Get all geometries that have at least one expire config defined + * from the database and return the result set. + */ + pg_result_t get_geoms_by_id(osmium::item_type type, osmid_t id) const; void flush() { m_copy_mgr.flush(); } diff -Nru osm2pgsql-1.8.1+ds/src/flex-write.cpp osm2pgsql-1.9.0+ds/src/flex-write.cpp --- osm2pgsql-1.8.1+ds/src/flex-write.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-write.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -253,8 +254,8 @@ void flex_write_column(lua_State *lua_state, db_copy_mgr_t *copy_mgr, - flex_table_column_t const &column, expire_tiles *expire, - expire_config_t const &expire_config) + flex_table_column_t const &column, + std::vector *expire) { // If there is nothing on the Lua stack, then the Lua function add_row() // was called without a table parameter. In that case this column will @@ -430,12 +431,12 @@ type == table_column_type::multilinestring || type == table_column_type::multipolygon); if (geom->srid() == column.srid()) { - expire->from_geometry_if_3857(*geom, expire_config); + column.do_expire(*geom, expire); copy_mgr->add_hex_geom(geom_to_ewkb(*geom, wrap_multi)); } else { auto const &proj = get_projection(column.srid()); auto const tgeom = geom::transform(*geom, proj); - expire->from_geometry_if_3857(tgeom, expire_config); + column.do_expire(tgeom, expire); copy_mgr->add_hex_geom(geom_to_ewkb(tgeom, wrap_multi)); } } else { @@ -462,7 +463,7 @@ void flex_write_row(lua_State *lua_state, table_connection_t *table_connection, osmium::item_type id_type, osmid_t id, geom::geometry_t const &geom, int srid, - expire_tiles *expire, expire_config_t const &expire_config) + std::vector *expire) { assert(table_connection); table_connection->new_line(); @@ -507,8 +508,7 @@ copy_mgr->add_column(area); } } else { - flex_write_column(lua_state, copy_mgr, column, expire, - expire_config); + flex_write_column(lua_state, copy_mgr, column, expire); } } diff -Nru osm2pgsql-1.8.1+ds/src/flex-write.hpp osm2pgsql-1.9.0+ds/src/flex-write.hpp --- osm2pgsql-1.8.1+ds/src/flex-write.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/flex-write.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -12,7 +12,8 @@ #include "expire-tiles.hpp" #include "flex-table.hpp" -#include "lua.hpp" + +#include #include @@ -32,12 +33,12 @@ void flex_write_column(lua_State *lua_state, db_copy_mgr_t *copy_mgr, - flex_table_column_t const &column, expire_tiles *expire, - expire_config_t const &expire_config); + flex_table_column_t const &column, + std::vector *expire); void flex_write_row(lua_State *lua_state, table_connection_t *table_connection, osmium::item_type id_type, osmid_t id, geom::geometry_t const &geom, int srid, - expire_tiles *expire, expire_config_t const &expire_config); + std::vector *expire); #endif // OSM2PGSQL_FLEX_WRITE_HPP diff -Nru osm2pgsql-1.8.1+ds/src/format.hpp osm2pgsql-1.9.0+ds/src/format.hpp --- osm2pgsql-1.8.1+ds/src/format.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/format.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -15,8 +15,9 @@ #include -template -std::runtime_error fmt_error(S const &format_str, TArgs &&...args) +template +std::runtime_error fmt_error(fmt::format_string format_str, + TArgs &&...args) { return std::runtime_error{ fmt::format(format_str, std::forward(args)...)}; diff -Nru osm2pgsql-1.8.1+ds/src/gazetteer-style.hpp osm2pgsql-1.9.0+ds/src/gazetteer-style.hpp --- osm2pgsql-1.8.1+ds/src/gazetteer-style.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gazetteer-style.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,6 +10,7 @@ * For a full list of authors see the git log. */ +#include #include #include #include @@ -85,8 +86,10 @@ explicit gazetteer_copy_mgr_t( std::shared_ptr const &processor) : db_copy_mgr_t(processor), - m_table(std::make_shared("place", "place_id")) - {} + m_table( + std::make_shared("public", "place", "place_id")) + { + } void prepare() { new_line(m_table); } diff -Nru osm2pgsql-1.8.1+ds/src/gen/canvas.cpp osm2pgsql-1.9.0+ds/src/gen/canvas.cpp --- osm2pgsql-1.8.1+ds/src/gen/canvas.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/canvas.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,130 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "canvas.hpp" +#include "raster.hpp" + +cimg_library::CImg canvas_t::create_pointlist(geom::point_list_t const &pl, + tile_t const &tile) const +{ + cimg_library::CImg points{static_cast(pl.size()), 2}; + + int n = 0; + for (auto const point : pl) { + auto const tp = tile.to_tile_coords(point, m_extent); + points(n, 0) = static_cast(static_cast(m_buffer) + tp.x()); + points(n, 1) = + static_cast(static_cast(m_buffer + m_extent) - tp.y()); + ++n; + } + + return points; +} + +std::size_t canvas_t::draw_polygon(geom::polygon_t const &polygon, + tile_t const &tile) +{ + if (polygon.inners().empty()) { + m_rast.draw_polygon(create_pointlist(polygon.outer(), tile), &White); + return polygon.outer().size(); + } + + std::size_t num_points = polygon.outer().size(); + m_temp.draw_polygon(create_pointlist(polygon.outer(), tile), &White); + for (auto const &inner : polygon.inners()) { + num_points += inner.size(); + m_temp.draw_polygon(create_pointlist(inner, tile), &Black); + } + m_rast |= m_temp; + + return num_points; +} + +std::size_t canvas_t::draw_linestring(geom::linestring_t const &linestring, + tile_t const &tile) +{ + m_rast.draw_line(create_pointlist(linestring, tile), &White); + return linestring.size(); +} + +std::size_t canvas_t::draw(geom::geometry_t const &geometry, tile_t const &tile) +{ + if (geometry.is_linestring()) { + auto const &linestring = geometry.get(); + return draw_linestring(linestring, tile); + } + + if (geometry.is_polygon()) { + auto const &polygon = geometry.get(); + return draw_polygon(polygon, tile); + } + + if (geometry.is_multipolygon()) { + auto const &mp = geometry.get(); + std::size_t num_points = 0; + for (auto const &p : mp) { + num_points += draw_polygon(p, tile); + } + return num_points; + } + + // XXX other geometry types? + + return 0; +} + +void canvas_t::save(std::string const &filename) const +{ + m_rast.save(filename.c_str()); +} + +std::string canvas_t::to_wkb(tile_t const &tile, double margin) const +{ + std::string wkb; + wkb.reserve(61 + 2 + m_rast.size()); + + // header + wkb_raster_header header{}; + header.nBands = 1; + header.scaleX = tile.extent() / static_cast(m_extent); + header.scaleY = -header.scaleX; + header.ipX = tile.xmin(margin); + header.ipY = tile.ymax(margin); + header.width = m_extent + 2 * m_buffer; + header.height = header.width; + add_raster_header(&wkb, header); + + // band + wkb_raster_band band{}; + band.bits = 4; + add_raster_band(&wkb, band); + + // rasterdata + wkb.append(reinterpret_cast(m_rast.data()), m_rast.size()); + + assert(wkb.size() == 61 + 2 + m_rast.size()); + + return wkb; +} + +void canvas_t::merge(canvas_t const &other) { m_rast |= other.m_rast; } + +std::string to_hex(std::string const &in) +{ + std::string result; + char const *const lookup_hex = "0123456789ABCDEF"; + + for (const auto c : in) { + unsigned int const num = static_cast(c); + result += lookup_hex[(num >> 4U) & 0xfU]; + result += lookup_hex[num & 0xfU]; + } + + return result; +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/canvas.hpp osm2pgsql-1.9.0+ds/src/gen/canvas.hpp --- osm2pgsql-1.8.1+ds/src/gen/canvas.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/canvas.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,88 @@ +#ifndef OSM2PGSQL_CANVAS_HPP +#define OSM2PGSQL_CANVAS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "geom.hpp" +#include "tile.hpp" + +#define cimg_display 0 // NOLINT(cppcoreguidelines-macro-usage) +#include "CImg.h" + +#include + +/** + * This class wraps the image class from the CImg library. + */ +class canvas_t +{ +public: + static void info() { cimg_library::cimg::info(); } + + /** + * Create a new image canvas. It will be quadratic and have the width and + * height extent + 2*buffer. + */ + canvas_t(std::size_t extent, std::size_t buffer) + : m_extent(extent), + m_buffer(buffer), m_rast{size(), size(), 1, 1, 0}, m_temp{size(), size(), + 1, 1, 0} + {} + + unsigned int size() const noexcept + { + return static_cast(m_extent + 2 * m_buffer); + } + + unsigned char const *begin() const noexcept { return m_rast.begin(); } + unsigned char const *end() const noexcept { return m_rast.end(); } + + std::size_t draw(geom::geometry_t const &geometry, tile_t const &tile); + + unsigned char operator()(int x, int y) const noexcept + { + return m_rast(x, y, 0, 0); + } + + void open_close(unsigned int buffer_size) + { + m_rast.dilate(buffer_size).erode(buffer_size * 2).dilate(buffer_size); + } + + void save(std::string const &filename) const; + + std::string to_wkb(tile_t const &tile, double margin) const; + + void merge(canvas_t const &other); + +private: + constexpr static unsigned char const Black = 0; + constexpr static unsigned char const White = 255; + + using image_type = cimg_library::CImg; + + cimg_library::CImg create_pointlist(geom::point_list_t const &pl, + tile_t const &tile) const; + + std::size_t draw_polygon(geom::polygon_t const &polygon, + tile_t const &tile); + + std::size_t draw_linestring(geom::linestring_t const &linestring, + tile_t const &tile); + + std::size_t m_extent; + std::size_t m_buffer; + image_type m_rast; + image_type m_temp; +}; // class canvas_t + +std::string to_hex(std::string const &in); + +#endif // OSM2PGSQL_CANVAS_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-base.cpp osm2pgsql-1.9.0+ds/src/gen/gen-base.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-base.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-base.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,146 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include "format.hpp" +#include "params.hpp" + +#include + +gen_base_t::gen_base_t(pg_conn_t *connection, params_t *params) +: m_connection(connection), m_params(params) +{ + assert(connection); + assert(params); + + params->check_identifier_with_default("schema", ""); + auto const schema = params->get_identifier("schema"); + + if (params->has("src_table")) { + auto const src_table = get_params().get_identifier("src_table"); + params->set("src", qualified_name(schema, src_table)); + } + + if (params->has("dest_table")) { + auto const dest_table = get_params().get_identifier("dest_table"); + params->set("dest", qualified_name(schema, dest_table)); + } + + if (!params->has("geom_column")) { + params->set("geom_column", "geom"); + } + + m_debug = get_params().get_bool("debug", false); +} + +void gen_base_t::check_src_dest_table_params_exist() +{ + if (!m_params->has("src_table")) { + throw fmt_error("Missing 'src_table' parameter in generalizer{}.", + context()); + } + + if (!m_params->has("dest_table")) { + throw fmt_error("Missing 'dest_table' parameter in generalizer{}.", + context()); + } + + if (m_params->get("src_table") == m_params->get("dest_table")) { + throw fmt_error("The 'src_table' and 'dest_table' parameters " + "must be different in generalizer{}.", + context()); + } +} + +void gen_base_t::check_src_dest_table_params_same() +{ + if (!m_params->has("src_table")) { + throw fmt_error("Missing 'src_table' parameter in generalizer{}.", + context()); + } + + if (m_params->has("dest_table") && + m_params->get("dest_table") != m_params->get("src_table")) { + throw fmt_error("The 'dest_table' parameter must be the same " + "as 'src_table' if it exists in generalizer{}.", + context()); + } +} + +std::string gen_base_t::name() { return get_params().get_string("name", ""); } + +std::string gen_base_t::context() +{ + auto const gen_name = name(); + return gen_name.empty() ? "" : fmt::format(" '{}'", gen_name); +} + +static pg_result_t dbexec_internal( + pg_conn_t const &connection, std::string const &templ, + fmt::dynamic_format_arg_store const &format_store) +{ + try { + auto const sql = fmt::vformat(templ, format_store); + return connection.exec(sql); + } catch (fmt::format_error const &e) { + log_error("Missing parameter for template: '{}'", templ); + throw; + } +} + +pg_result_t gen_base_t::dbexec(std::string const &templ) +{ + fmt::dynamic_format_arg_store format_store; + for (auto const &[key, value] : get_params()) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + return dbexec_internal(connection(), templ, format_store); +} + +pg_result_t gen_base_t::dbexec(params_t const &tmp_params, + std::string const &templ) +{ + fmt::dynamic_format_arg_store format_store; + for (auto const &[key, value] : get_params()) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + for (auto const &[key, value] : tmp_params) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + return dbexec_internal(connection(), templ, format_store); +} + +void gen_base_t::raster_table_preprocess(std::string const &table) +{ + params_t tmp_params; + tmp_params.set("TABLE", table); + + dbexec(tmp_params, "SELECT DropRasterConstraints('{schema}'::name," + " '{TABLE}'::name, 'rast'::name)"); +} + +void gen_base_t::raster_table_postprocess(std::string const &table) +{ + params_t tmp_params; + tmp_params.set("TABLE", table); + + dbexec(tmp_params, R"(SELECT AddRasterConstraints('{schema}'::name,)" + R"( '{TABLE}'::name, 'rast'::name))"); + dbexec(tmp_params, R"(ALTER TABLE "{schema}"."{TABLE}")" + R"( VALIDATE CONSTRAINT enforce_max_extent_rast)"); + dbexec(tmp_params, R"(ANALYZE "{schema}"."{TABLE}")"); +} + +void gen_base_t::merge_timers(gen_base_t const &other) +{ + for (std::size_t n = 0; n < m_timers.size(); ++n) { + m_timers[n] += other.m_timers[n]; + } +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-base.hpp osm2pgsql-1.9.0+ds/src/gen/gen-base.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-base.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-base.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,112 @@ +#ifndef OSM2PGSQL_GEN_BASE_HPP +#define OSM2PGSQL_GEN_BASE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "logging.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include +#include +#include + +class params_t; +class tile_t; + +/** + * Base class for generalization strategies. + */ +class gen_base_t +{ +public: + virtual ~gen_base_t() = default; + + /// Process data. Used for non-tile-based generalizers. + virtual void process() {} + + /// Process one tile. Used for tile-based generalizers. + virtual void process(tile_t const & /*tile*/) {} + + /// Optional postprocessing after all tiles. + virtual void post() {} + + /// Get the name of the generalization strategy. + virtual std::string_view strategy() const noexcept = 0; + + virtual bool on_tiles() const noexcept { return false; } + + virtual uint32_t get_zoom() const noexcept { return 0; } + + void merge_timers(gen_base_t const &other); + + std::vector const &timers() const noexcept + { + return m_timers; + } + + bool debug() const noexcept { return m_debug; } + + std::string name(); + + std::string context(); + + template + void log_gen(ARGS... args) + { + if (m_debug) { + log_debug(args...); + } + } + +protected: + gen_base_t(pg_conn_t *connection, params_t *params); + + /** + * Check that the 'src_table' and 'dest_table' parameters exist and that + * they are different. + */ + void check_src_dest_table_params_exist(); + + /** + * Check that the 'src_table' parameter exists. If the 'dest_table' + * parameter exists it must be the same as 'src_table'. + */ + void check_src_dest_table_params_same(); + + pg_conn_t &connection() noexcept { return *m_connection; } + + std::size_t add_timer(char const *name) + { + m_timers.emplace_back(name); + return m_timers.size() - 1; + } + + util::timer_t &timer(std::size_t n) noexcept { return m_timers[n]; } + + params_t const &get_params() const noexcept { return *m_params; } + + pg_result_t dbexec(std::string const &templ); + + pg_result_t dbexec(params_t const &tmp_params, std::string const &templ); + + void raster_table_preprocess(std::string const &table); + + void raster_table_postprocess(std::string const &table); + +private: + std::vector m_timers; + pg_conn_t *m_connection; + params_t *m_params; + bool m_debug = false; +}; // class gen_base_t + +#endif // OSM2PGSQL_GEN_BASE_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-create.cpp osm2pgsql-1.9.0+ds/src/gen/gen-create.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-create.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-create.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,45 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-create.hpp" + +#include "gen-discrete-isolation.hpp" +#include "gen-rivers.hpp" +#include "gen-tile-builtup.hpp" +#include "gen-tile-raster.hpp" +#include "gen-tile-vector.hpp" +#include "params.hpp" + +std::unique_ptr create_generalizer(std::string const &strategy, + pg_conn_t *connection, + params_t *params) +{ + auto generalizer = [&]() -> std::unique_ptr { + if (strategy == "builtup") { + return std::make_unique(connection, params); + } + if (strategy == "discrete-isolation") { + return std::make_unique(connection, params); + } + if (strategy == "raster-union") { + return std::make_unique(connection, + params); + } + if (strategy == "rivers") { + return std::make_unique(connection, params); + } + if (strategy == "vector-union") { + return std::make_unique(connection, + params); + } + throw fmt_error("Unknown generalization strategy '{}'.", strategy); + }(); + + return generalizer; +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-create.hpp osm2pgsql-1.9.0+ds/src/gen/gen-create.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-create.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-create.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,26 @@ +#ifndef OSM2PGSQL_GEN_CREATE_HPP +#define OSM2PGSQL_GEN_CREATE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include +#include + +class params_t; +class pg_conn_t; + +/// Instantiate a generalizer for the specified strategy. +std::unique_ptr create_generalizer(std::string const &strategy, + pg_conn_t *connection, + params_t *params); + +#endif // OSM2PGSQL_GEN_CREATE_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-discrete-isolation.cpp osm2pgsql-1.9.0+ds/src/gen/gen-discrete-isolation.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-discrete-isolation.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-discrete-isolation.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,149 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-discrete-isolation.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include + +gen_di_t::gen_di_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_get(add_timer("get")), + m_timer_sort(add_timer("sort")), m_timer_di(add_timer("di")), + m_timer_reorder(add_timer("reorder")), m_timer_write(add_timer("write")) +{ + check_src_dest_table_params_same(); + + params->check_identifier_with_default("id_column", "id"); + params->check_identifier_with_default("importance_column", "importance"); +} + +void gen_di_t::process() +{ + struct feature + { + // input: unique id of the feature + uint64_t id; + + // input: importance of the feature (positive, larger is more imporant) + double importance; + + // input: x/y coordinate of the feature + double x; + double y; + + // output: discrete isolation + double di; + + // output: rank for importance + uint32_t irank; + }; + + log_gen("Reading data from database..."); + + std::vector data; + timer(m_timer_get).start(); + { + auto const result = dbexec(R"( +SELECT {id_column}, {importance_column}, + ST_X({geom_column}), ST_Y({geom_column}) +FROM {src} WHERE {importance_column} > 0 +)"); + + data.reserve(result.num_tuples()); + for (int i = 0; i < result.num_tuples(); ++i) { + data.push_back({std::strtoull(result.get_value(i, 0), nullptr, 10), + std::strtod(result.get_value(i, 1), nullptr), + std::strtod(result.get_value(i, 2), nullptr), + std::strtod(result.get_value(i, 3), nullptr), 0.0, + 0}); + } + } + timer(m_timer_get).stop(); + log_gen("Read {} features", data.size()); + + if (data.size() < 2) { + log_gen("Found fewer than two features. Nothing to do."); + return; + } + + log_gen("Sorting data by importance..."); + timer(m_timer_sort).start(); + { + std::sort(data.begin(), data.end(), + [](feature const &a, feature const &b) noexcept { + return a.importance > b.importance; + }); + { + uint32_t n = 0; + for (auto &item : data) { + item.irank = n++; + } + } + } + timer(m_timer_sort).stop(); + + log_gen("Calculating discrete isolation..."); + timer(m_timer_di).start(); + { + std::vector> coords; + coords.reserve(data.size()); + for (auto const &d : data) { + coords.emplace_back(d.x, d.y); + } + + for (std::size_t n = 1; n < data.size(); ++n) { + if (n % 10000 == 0) { + log_gen(" {}", n); + } + double min = 100000000000000.0; + for (std::size_t m = 0; m < n; ++m) { + double const dx = coords[m].first - coords[n].first; + double const dy = coords[m].second - coords[n].second; + double const dist = dx * dx + dy * dy; + if (dist < min) { + min = dist; + } + } + data[n].di = sqrt(min); + } + data[0].di = data[1].di + 1; + } + timer(m_timer_di).stop(); + + log_gen("Sorting data by discrete isolation..."); + timer(m_timer_reorder).start(); + std::sort(data.begin(), data.end(), + [](feature const &a, feature const &b) noexcept { + return a.di > b.di; + }); + timer(m_timer_reorder).stop(); + + log_gen("Writing results to destination table..."); + dbexec("PREPARE update (int, real, int4, int8) AS" + " UPDATE {src} SET dirank = $1, discr_iso = $2, irank = $3" + " WHERE {id_column} = $4"); + + timer(m_timer_write).start(); + connection().exec("BEGIN"); + std::size_t n = 0; + for (auto const &d : data) { + connection().exec_prepared("update", n++, d.di, d.irank, d.id); + } + connection().exec("COMMIT"); + timer(m_timer_write).stop(); + + dbexec("ANALYZE {src}"); + + log_gen("Done."); +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-discrete-isolation.hpp osm2pgsql-1.9.0+ds/src/gen/gen-discrete-isolation.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-discrete-isolation.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-discrete-isolation.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,37 @@ +#ifndef OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP +#define OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include + +class gen_di_t : public gen_base_t +{ +public: + gen_di_t(pg_conn_t *connection, params_t *params); + + void process() override; + + std::string_view strategy() const noexcept override + { + return "discrete-isolation"; + } + +private: + std::size_t m_timer_get; + std::size_t m_timer_sort; + std::size_t m_timer_di; + std::size_t m_timer_reorder; + std::size_t m_timer_write; +}; + +#endif // OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-rivers.cpp osm2pgsql-1.9.0+ds/src/gen/gen-rivers.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-rivers.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-rivers.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,349 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-rivers.hpp" + +#include "geom-functions.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "util.hpp" +#include "wkb.hpp" + +#include +#include +#include +#include +#include + +gen_rivers_t::gen_rivers_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_area(add_timer("area")), + m_timer_prep(add_timer("prep")), m_timer_get(add_timer("get")), + m_timer_sort(add_timer("sort")), m_timer_net(add_timer("net")), + m_timer_remove(add_timer("remove")), m_timer_width(add_timer("width")), + m_timer_write(add_timer("write")), + m_delete_existing(params->has("delete_existing")) +{ + check_src_dest_table_params_exist(); + + params->check_identifier_with_default("src_areas", "waterway_areas"); + params->check_identifier_with_default("id_column", "way_id"); + params->check_identifier_with_default("width_column", "width"); + params->check_identifier_with_default("name_column", "name"); + + params->set("qualified_src_areas", + qualified_name(get_params().get_string("schema"), + get_params().get_string("src_areas"))); +} + +/// The data for a graph edge in the waterway network. +struct edge_t +{ + // All the points in this edge + geom::linestring_t points; + + // Edges can be made from (part) of one or more OSM ways, this is the id + // of one of them. + osmid_t id = 0; + + // The width of the river along this edge + double width = 0.0; +}; + +bool operator<(edge_t const &a, edge_t const &b) noexcept +{ + assert(a.points.size() > 1 && b.points.size() > 1); + if (a.points[0] == b.points[0]) { + return a.points[1] < b.points[1]; + } + return a.points[0] < b.points[0]; +} + +bool operator<(edge_t const &a, geom::point_t b) noexcept +{ + assert(!a.points.empty()); + return a.points[0] < b; +} + +bool operator<(geom::point_t a, edge_t const &b) noexcept +{ + assert(!b.points.empty()); + return a < b.points[0]; +} + +static void +follow_chain_and_set_width(edge_t const &edge, std::vector *edges, + std::map const &node_order, + geom::linestring_t *seen) +{ + assert(!edge.points.empty()); + + auto const seen_it = + std::find(seen->cbegin(), seen->cend(), edge.points[0]); + if (seen_it != seen->cend()) { + return; // loop detected + } + + seen->push_back(edge.points[0]); + + assert(edge.points.size() > 1); + auto const next_point = edge.points.back(); + if (node_order.at(next_point) > 1) { + auto const [s, e] = + std::equal_range(edges->begin(), edges->end(), next_point); + + if (std::next(s) == e) { + if (s->width < edge.width) { + s->width = edge.width; + follow_chain_and_set_width(*s, edges, node_order, seen); + } + } else { + for (auto it = s; it != e; ++it) { + assert(it->points[0] == next_point); + if (it->width < edge.width) { + it->width = edge.width; + auto seen2 = *seen; + follow_chain_and_set_width(*it, edges, node_order, &seen2); + } + } + } + } +} + +static void assemble_edge(edge_t *edge, std::vector *edges, + std::map const &node_order) + +{ + assert(edge); + assert(edges); + while (true) { + assert(edge->points.size() > 1); + geom::point_t const next_point = edge->points.back(); + + auto const count = node_order.at(next_point); + if (count != 2) { + return; + } + + auto const [s, e] = + std::equal_range(edges->begin(), edges->end(), next_point); + + if (s == e) { + return; + } + assert(e == std::next(s)); + + auto const it = s; + if (it->points.size() == 1 || &*it == edge) { + return; + } + + if (it->points[0] != next_point) { + return; + } + assert(it != edges->end()); + + edge->width = std::max(edge->width, it->width); + + if (it->points.size() == 2) { + edge->points.push_back(it->points.back()); + it->points.resize(1); + it->points.shrink_to_fit(); + } else { + edge->points.insert(edge->points.end(), + std::next(it->points.begin()), + it->points.end()); + it->points.resize(1); + it->points.shrink_to_fit(); + return; + } + } +} + +/// Get some stats from source table +void gen_rivers_t::get_stats() +{ + auto const result = + dbexec("SELECT count(*), sum(ST_NumPoints(geom)) FROM {src}"); + + m_num_waterways = strtoul(result.get_value(0, 0), nullptr, 10); + m_num_points = strtoul(result.get_value(0, 1), nullptr, 10); + + log_gen("Found {} waterways with {} points.", m_num_waterways, + m_num_points); +} + +static std::string const & +get_name(std::unordered_map const &names, osmid_t id) +{ + static std::string const empty; + auto const it = names.find(id); + if (it == names.end()) { + return empty; + } + return it->second; +} + +void gen_rivers_t::process() +{ + log_gen("Calculate waterway area width..."); + timer(m_timer_area).start(); + dbexec(R"(UPDATE {qualified_src_areas} SET width =)" + R"( (ST_MaximumInscribedCircle("{geom_column}")).radius * 2)" + R"( WHERE width IS NULL)"); + dbexec("ANALYZE {qualified_src_areas}"); + timer(m_timer_area).stop(); + + log_gen("Get 'width' from areas onto lines..."); + timer(m_timer_prep).start(); + dbexec(R"( +WITH _covered_lines AS ( + SELECT "{geom_column}" AS geom, "{id_column}" AS wid FROM {src} w + WHERE ST_NumPoints(w."{geom_column}") > 2 AND ST_CoveredBy(w."{geom_column}", + (SELECT ST_Union("{geom_column}") FROM {qualified_src_areas} a + WHERE ST_Intersects(w."{geom_column}", a."{geom_column}"))) +), _intersections AS ( + SELECT w.wid, ST_Intersection(a.geom, w.geom) AS inters, + ST_Length(w.geom) AS wlength, a.width AS width + FROM _covered_lines w, {qualified_src_areas} a + WHERE ST_Intersects(w.geom, a.geom) +), _lines AS ( + SELECT wid, wlength, ST_Length(inters) * width AS lenwidth FROM _intersections + WHERE ST_GeometryType(inters) IN ('ST_LineString', 'ST_MultiLineString') +), _glines AS ( + SELECT wid, sum(lenwidth) / wlength AS width FROM _lines + GROUP BY wid, wlength +) +UPDATE {src} a SET width = l.width + FROM _glines l WHERE l.wid = a."{id_column}" AND a.width IS NULL + )"); + timer(m_timer_prep).stop(); + + log_gen("Reading waterway lines from database..."); + get_stats(); + + // This vector will initially contain all segments (connection between + // two points) from waterway ways. They will later be assembled into + // graph edges connecting points where the waterways network branches. + std::vector edges; + edges.reserve(m_num_points - m_num_waterways); + + // This stores the order of each node in our graph, i.e. the number of + // connections this node has. Order 1 are beginning or end of a waterway, + // order 2 is just the continuing waterway, order >= 3 is a branching + // point. + std::map node_order; + + // This is where we keep the names of all waterways indexed by their + // way id. + std::unordered_map names; + + timer(m_timer_get).start(); + { + auto const result = dbexec(R"( +SELECT "{id_column}", "{width_column}", "{name_column}", "{geom_column}" + FROM {src}; +)"); + + for (int i = 0; i < result.num_tuples(); ++i) { + auto const id = std::strtol(result.get_value(i, 0), nullptr, 10); + auto const width = std::strtod(result.get_value(i, 1), nullptr); + auto const name = result.get(i, 2); + if (!name.empty()) { + names.emplace(id, name); + } + auto const geom = ewkb_to_geom(decode_hex(result.get_value(i, 3))); + + if (geom.is_linestring()) { + auto const &ls = geom.get(); + geom::for_each_segment(ls, + [&](geom::point_t a, geom::point_t b) { + if (a != b) { + auto &f = edges.emplace_back(); + f.points.push_back(a); + f.points.push_back(b); + f.id = id; + f.width = width; + node_order[a]++; + node_order[b]++; + } + }); + } + } + } + timer(m_timer_get).stop(); + log_gen("Read {} segments, {} unique points, and {} names.", edges.size(), + node_order.size(), names.size()); + + if (edges.size() < 2) { + log_gen("Found fewer than two segments. Nothing to do."); + return; + } + + log_gen("Sorting segments..."); + timer(m_timer_sort).start(); + std::sort(edges.begin(), edges.end()); + timer(m_timer_sort).stop(); + + log_gen("Assembling edges from segments..."); + timer(m_timer_net).start(); + for (auto &edge : edges) { + if (edge.points.size() > 1) { + assemble_edge(&edge, &edges, node_order); + } + } + timer(m_timer_net).stop(); + + log_gen("Removing now empty edges..."); + timer(m_timer_remove).start(); + { + auto const last = + std::remove_if(edges.begin(), edges.end(), [](edge_t const &edge) { + return edge.points.size() == 1; + }); + edges.erase(last, edges.end()); + std::sort(edges.begin(), edges.end()); + } + timer(m_timer_remove).stop(); + + log_gen("Network has {} edges.", edges.size()); + + log_gen("Propagating 'width' property downstream..."); + timer(m_timer_width).start(); + for (auto &edge : edges) { + assert(!edge.points.empty()); + geom::linestring_t seen; + follow_chain_and_set_width(edge, &edges, node_order, &seen); + } + timer(m_timer_width).stop(); + + if (m_delete_existing) { + dbexec("TRUNCATE {dest}"); + } + + log_gen("Writing results to destination table..."); + dbexec("PREPARE ins (int8, real, text, geometry) AS" + " INSERT INTO {dest} ({id_column}, width, name, geom)" + " VALUES ($1, $2, $3, $4)"); + + timer(m_timer_write).start(); + connection().exec("BEGIN"); + for (auto &edge : edges) { + geom::geometry_t const geom{std::move(edge.points), 3857}; + auto const wkb = geom_to_ewkb(geom); + connection().exec_prepared("ins", edge.id, edge.width, + get_name(names, edge.id), binary_param(wkb)); + } + connection().exec("COMMIT"); + timer(m_timer_write).stop(); + + dbexec("ANALYZE {dest}"); + + log_gen("Done."); +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-rivers.hpp osm2pgsql-1.9.0+ds/src/gen/gen-rivers.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-rivers.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-rivers.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,43 @@ +#ifndef OSM2PGSQL_GEN_RIVERS_HPP +#define OSM2PGSQL_GEN_RIVERS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include + +class gen_rivers_t : public gen_base_t +{ +public: + gen_rivers_t(pg_conn_t *connection, params_t *params); + + void process() override; + + std::string_view strategy() const noexcept override { return "rivers"; } + +private: + void get_stats(); + + std::size_t m_timer_area; + std::size_t m_timer_prep; + std::size_t m_timer_get; + std::size_t m_timer_sort; + std::size_t m_timer_net; + std::size_t m_timer_remove; + std::size_t m_timer_width; + std::size_t m_timer_write; + + std::size_t m_num_waterways = 0; + std::size_t m_num_points = 0; + bool m_delete_existing; +}; + +#endif // OSM2PGSQL_GEN_RIVERS_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile-builtup.cpp osm2pgsql-1.9.0+ds/src/gen/gen-tile-builtup.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile-builtup.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile-builtup.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,280 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-builtup.hpp" + +#include "canvas.hpp" +#include "geom-functions.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "raster.hpp" +#include "tile.hpp" +#include "tracer.hpp" +#include "wkb.hpp" + +#include + +static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept +{ + return ((value + multiple - 1U) / multiple) * multiple; +} + +gen_tile_builtup_t::gen_tile_builtup_t(pg_conn_t *connection, params_t *params) +: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")), + m_timer_simplify(add_timer("simplify")), + m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write")) +{ + check_src_dest_table_params_exist(); + + m_source_tables = + osmium::split_string(get_params().get_string("src_tables"), ','); + + m_margin = get_params().get_double("margin"); + m_image_extent = uint_in_range(*params, "image_extent", 1024, 65536, 2048); + m_image_buffer = + uint_in_range(*params, "image_buffer", 0, m_image_extent, 0); + + auto const buffer_sizes = + osmium::split_string(get_params().get_string("buffer_size"), ','); + for (auto const &bs : buffer_sizes) { + m_buffer_sizes.push_back(std::strtoul(bs.c_str(), nullptr, 10)); + } + + m_turdsize = static_cast( + uint_in_range(*params, "turdsize", 0, 65536, m_turdsize)); + m_min_area = get_params().get_double("min_area", 0.0); + + if (get_params().has("area_column")) { + m_has_area_column = true; + get_params().get_identifier("area_column"); + } + + if (get_params().has("img_path")) { + m_image_path = get_params().get_string("img_path"); + } + + if (get_params().has("img_table")) { + m_image_table = get_params().get_string("img_table"); + + for (auto const &table : m_source_tables) { + for (char const variant : {'i', 'o'}) { + auto const table_name = + fmt::format("{}_{}_{}", m_image_table, table, variant); + connection->exec(R"( +CREATE TABLE IF NOT EXISTS "{}" ( + id SERIAL PRIMARY KEY NOT NULL, + zoom INT4, + x INT4, + y INT4, + rast RASTER +) +)", + table_name); + raster_table_preprocess(table_name); + } + } + } + + if (params->get_bool("make_valid")) { + params->set( + "geom_sql", + "(ST_Dump(ST_CollectionExtract(ST_MakeValid($1), 3))).geom"); + } else { + params->set("geom_sql", "$1"); + } + + if ((m_image_extent & (m_image_extent - 1)) != 0) { + throw fmt_error( + "The 'image_extent' parameter on generalizer{} must be power of 2.", + context()); + } + + m_image_buffer = + round_up(static_cast(m_margin * + static_cast(m_image_extent)), + 64U); + m_margin = static_cast(m_image_buffer) / + static_cast(m_image_extent); + + log_gen("Image extent: {}px, buffer: {}px, margin: {}", m_image_extent, + m_image_buffer, m_margin); + + int n = 0; + auto const schema = get_params().get_string("schema"); + for (auto const &src_table : m_source_tables) { + params_t tmp_params; + tmp_params.set("N", std::to_string(n++)); + tmp_params.set("SRC", qualified_name(schema, src_table)); + + dbexec(tmp_params, R"( +PREPARE get_geoms_{N} (real, real, real, real) AS + SELECT "{geom_column}", '' AS param + FROM {SRC} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"); + } + + if (m_has_area_column) { + dbexec(R"( +PREPARE insert_geoms (geometry, int, int) AS + INSERT INTO {dest} ("{geom_column}", x, y, "{area_column}") + VALUES ({geom_sql}, $2, $3, $4) +)"); + } else { + dbexec(R"( +PREPARE insert_geoms (geometry, int, int) AS + INSERT INTO {dest} ("{geom_column}", x, y) + VALUES ({geom_sql}, $2, $3) +)"); + } +} + +static void save_image_to_table(pg_conn_t *connection, canvas_t const &canvas, + tile_t const &tile, double margin, + std::string const &table, char const *variant, + std::string const &table_prefix) +{ + auto const wkb = to_hex(canvas.to_wkb(tile, margin)); + + connection->exec("INSERT INTO \"{}_{}_{}\" (zoom, x, y, rast)" + " VALUES ({}, {}, {}, '{}')", + table_prefix, table, variant, tile.zoom(), tile.x(), + tile.y(), wkb); +} + +namespace { + +struct param_canvas_t +{ + canvas_t canvas; + std::string table; +}; + +} // anonymous namespace + +using canvas_list_t = std::vector; + +static void draw_from_db(double margin, canvas_list_t *canvas_list, + pg_conn_t *conn, tile_t const &tile) +{ + int prep = 0; + auto const box = tile.box(margin); + for (auto &cc : *canvas_list) { + std::string const statement = "get_geoms_" + fmt::to_string(prep++); + auto const result = + conn->exec_prepared(statement.c_str(), box.min_x(), box.min_y(), + box.max_x(), box.max_y()); + + for (int n = 0; n < result.num_tuples(); ++n) { + auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0))); + cc.canvas.draw(geom, tile); + } + } +} + +void gen_tile_builtup_t::process(tile_t const &tile) +{ + delete_existing(tile); + + canvas_list_t canvas_list; + for (auto const &table : m_source_tables) { + canvas_list.push_back( + {canvas_t{m_image_extent, m_image_buffer}, table}); + } + + if (canvas_list.empty()) { + throw std::runtime_error{"No source tables?!"}; + } + + log_gen("Read from database and draw polygons..."); + timer(m_timer_draw).start(); + draw_from_db(m_margin, &canvas_list, &connection(), tile); + timer(m_timer_draw).stop(); + + std::size_t n = 0; + for (auto &[canvas, table] : canvas_list) { + log_gen("Handling table='{}'", table); + + if (!m_image_path.empty()) { + // Save input images for debugging + save_image_to_file(canvas, tile, m_image_path, table, "i", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store input images in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, table, + "i", m_image_table); + } + + if (m_buffer_sizes[n] > 0) { + log_gen("Generalize (buffer={} Mercator units)...", + m_buffer_sizes[n] * tile.extent() / + static_cast(m_image_extent)); + timer(m_timer_simplify).start(); + canvas.open_close(m_buffer_sizes[n]); + timer(m_timer_simplify).stop(); + } + + if (!m_image_path.empty()) { + // Save output image for debugging + save_image_to_file(canvas, tile, m_image_path, table, "o", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store output image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, table, + "o", m_image_table); + } + + ++n; + } + + log_gen("Merge bitmaps..."); + for (std::size_t n = 1; n < canvas_list.size(); ++n) { + canvas_list[0].canvas.merge(canvas_list[n].canvas); + } + + tracer_t tracer{m_image_extent, m_image_buffer, m_turdsize}; + + log_gen("Vectorize..."); + timer(m_timer_vectorize).start(); + auto const geometries = + tracer.trace(canvas_list[0].canvas, tile, m_min_area); + timer(m_timer_vectorize).stop(); + + log_gen("Write geometries to destination table..."); + timer(m_timer_write).start(); + for (auto const &geom : geometries) { + auto const wkb = to_hex(geom_to_ewkb(geom)); + if (m_has_area_column) { + connection().exec_prepared("insert_geoms", wkb, tile.x(), tile.y(), + geom::area(geom)); + } else { + connection().exec_prepared("insert_geoms", wkb, tile.x(), tile.y()); + } + } + timer(m_timer_write).stop(); + log_gen("Inserted {} generalized polygons", geometries.size()); +} + +void gen_tile_builtup_t::post() +{ + if (!m_image_table.empty()) { + for (auto const &table : m_source_tables) { + for (char const variant : {'i', 'o'}) { + raster_table_postprocess( + fmt::format("{}_{}_{}", m_image_table, table, variant)); + } + } + } + dbexec("ANALYZE {dest}"); +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile-builtup.hpp osm2pgsql-1.9.0+ds/src/gen/gen-tile-builtup.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile-builtup.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile-builtup.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,50 @@ +#ifndef OSM2PGSQL_GEN_TILE_BUILTUP_HPP +#define OSM2PGSQL_GEN_TILE_BUILTUP_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include +#include +#include + +class gen_tile_builtup_t final : public gen_tile_t +{ +public: + gen_tile_builtup_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_builtup_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override { return "builtup"; } + +private: + std::size_t m_timer_draw; + std::size_t m_timer_simplify; + std::size_t m_timer_vectorize; + std::size_t m_timer_write; + + std::vector m_source_tables; + std::string m_image_path; + std::string m_image_table; + double m_margin = 0.0; + std::size_t m_image_extent = 2048; + std::size_t m_image_buffer = 0; + std::vector m_buffer_sizes; + int m_turdsize = 2; + double m_min_area = 0.0; + bool m_has_area_column; +}; + +#endif // OSM2PGSQL_GEN_TILE_BUILTUP_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile.cpp osm2pgsql-1.9.0+ds/src/gen/gen-tile.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,74 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +#include + +gen_tile_t::gen_tile_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_delete(add_timer("delete")), + m_zoom(parse_zoom()) +{ + m_with_group_by = !get_params().get_identifier("group_by_column").empty(); + + if (get_params().get_bool("delete_existing")) { + m_delete_existing = true; + dbexec("PREPARE del_geoms (int, int) AS" + " DELETE FROM {dest} WHERE x=$1 AND y=$2"); + } +} + +uint32_t gen_tile_t::parse_zoom() +{ + if (!get_params().has("zoom")) { + throw fmt_error("Missing 'zoom' parameter in generalizer{}.", + context()); + } + + auto const pval = get_params().get("zoom"); + + if (!std::holds_alternative(pval)) { + throw fmt_error( + "Invalid value '{}' for 'zoom' parameter in generalizer{}.", + get_params().get_string("zoom"), context()); + } + + auto const value = std::get(pval); + if (value < 0 || value > 20) { + throw fmt_error( + "Invalid value '{}' for 'zoom' parameter in generalizer{}.", value, + context()); + } + return static_cast(value); +} + +void gen_tile_t::delete_existing(tile_t const &tile) +{ + if (!m_delete_existing) { + return; + } + + if (debug()) { + log_gen("Delete geometries from destination table..."); + } + + timer(m_timer_delete).start(); + auto const result = + connection().exec_prepared("del_geoms", tile.x(), tile.y()); + timer(m_timer_delete).stop(); + + if (debug()) { + log_gen("Deleted {} rows.", result.affected_rows()); + } +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile.hpp osm2pgsql-1.9.0+ds/src/gen/gen-tile.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,41 @@ +#ifndef OSM2PGSQL_GEN_TILE_HPP +#define OSM2PGSQL_GEN_TILE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +/** + * Base class for generalizations based on tiles. + */ +class gen_tile_t : public gen_base_t +{ +public: + bool on_tiles() const noexcept override { return true; } + + uint32_t get_zoom() const noexcept override { return m_zoom; } + +protected: + gen_tile_t(pg_conn_t *connection, params_t *params); + + uint32_t parse_zoom(); + + void delete_existing(tile_t const &tile); + + bool with_group_by() const noexcept { return m_with_group_by; } + +private: + std::size_t m_timer_delete; + uint32_t m_zoom; + bool m_delete_existing = false; + bool m_with_group_by = false; +}; + +#endif // OSM2PGSQL_GEN_TILE_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile-raster.cpp osm2pgsql-1.9.0+ds/src/gen/gen-tile-raster.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile-raster.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile-raster.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,258 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-raster.hpp" + +#include "canvas.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "raster.hpp" +#include "tile.hpp" +#include "tracer.hpp" +#include "wkb.hpp" + +#include + +static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept +{ + return ((value + multiple - 1U) / multiple) * multiple; +} + +gen_tile_raster_union_t::gen_tile_raster_union_t(pg_conn_t *connection, + params_t *params) +: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")), + m_timer_simplify(add_timer("simplify")), + m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write")) +{ + check_src_dest_table_params_exist(); + + m_margin = get_params().get_double("margin"); + m_image_extent = uint_in_range(*params, "image_extent", 1024, 65536, 2048); + m_image_buffer = + uint_in_range(*params, "image_buffer", 0, m_image_extent, 0); + m_buffer_size = uint_in_range(*params, "buffer_size", 1, 65536, 10); + m_turdsize = static_cast( + uint_in_range(*params, "turdsize", 0, 65536, m_turdsize)); + + std::string const where_condition = get_params().get_string("where", ""); + + if (get_params().has("img_path")) { + m_image_path = get_params().get_string("img_path"); + } + + if (get_params().has("img_table")) { + m_image_table = get_params().get_string("img_table"); + + for (char const variant : {'i', 'o'}) { + auto const table_name = + fmt::format("{}_{}", m_image_table, variant); + connection->exec(R"( +CREATE TABLE IF NOT EXISTS "{}" ( + type TEXT, + zoom INT4, + x INT4, + y INT4, + rast RASTER +) +)", + table_name); + raster_table_preprocess(table_name); + } + } + + if (get_params().get_bool("make_valid")) { + params->set( + "geom_sql", + "(ST_Dump(ST_CollectionExtract(ST_MakeValid($1), 3))).geom"); + } else { + params->set("geom_sql", "$1"); + } + + if ((m_image_extent & (m_image_extent - 1)) != 0) { + throw fmt_error( + "The 'image_extent' parameter on generalizer{} must be power of 2.", + context()); + } + + m_image_buffer = + round_up(static_cast(m_margin * + static_cast(m_image_extent)), + 64U); + m_margin = static_cast(m_image_buffer) / + static_cast(m_image_extent); + + log_gen("Image extent: {}px, buffer: {}px, margin: {}", m_image_extent, + m_image_buffer, m_margin); + + std::string prepare; + if (with_group_by()) { + prepare = R"( +PREPARE get_geoms (real, real, real, real) AS + SELECT "{geom_column}", "{group_by_column}" + FROM {src} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"; + dbexec(R"( +PREPARE insert_geoms (geometry, int, int, text) AS + INSERT INTO {dest} ("{geom_column}", x, y, "{group_by_column}") + VALUES ({geom_sql}, $2, $3, $4) +)"); + } else { + prepare = R"( +PREPARE get_geoms (real, real, real, real) AS + SELECT "{geom_column}", NULL AS param + FROM {src} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"; + dbexec(R"( +PREPARE insert_geoms (geometry, int, int, text) AS + INSERT INTO {dest} ("{geom_column}", x, y) VALUES ({geom_sql}, $2, $3) +)"); + } + + if (!where_condition.empty()) { + prepare.append(fmt::format(" AND ({})", where_condition)); + } + + dbexec(prepare); +} + +static void save_image_to_table(pg_conn_t *connection, canvas_t const &canvas, + tile_t const &tile, double margin, + std::string const ¶m, char const *variant, + std::string const &table_prefix) +{ + auto const wkb = to_hex(canvas.to_wkb(tile, margin)); + + connection->exec("INSERT INTO \"{}_{}\" (type, zoom, x, y, rast)" + " VALUES ('{}', {}, {}, {}, '{}')", + table_prefix, variant, param, tile.zoom(), tile.x(), + tile.y(), wkb); +} + +namespace { + +struct param_canvas_t +{ + canvas_t canvas; + std::size_t points = 0; + + param_canvas_t(unsigned int image_extent, unsigned int image_buffer) + : canvas(image_extent, image_buffer) + {} +}; + +} // anonymous namespace + +using canvas_list_t = std::unordered_map; + +static void draw_from_db(double margin, unsigned int image_extent, + unsigned int image_buffer, canvas_list_t *canvas_list, + pg_conn_t *conn, tile_t const &tile) +{ + auto const box = tile.box(margin); + auto const result = conn->exec_prepared( + "get_geoms", box.min_x(), box.min_y(), box.max_x(), box.max_y()); + + for (int n = 0; n < result.num_tuples(); ++n) { + std::string param = result.get_value(n, 1); + auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0))); + + auto const [it, success] = canvas_list->try_emplace( + std::move(param), image_extent, image_buffer); + + it->second.points += it->second.canvas.draw(geom, tile); + } +} + +void gen_tile_raster_union_t::process(tile_t const &tile) +{ + delete_existing(tile); + + canvas_list_t canvas_list; + + log_gen("Read from database and draw polygons..."); + timer(m_timer_draw).start(); + draw_from_db(m_margin, m_image_extent, m_image_buffer, &canvas_list, + &connection(), tile); + timer(m_timer_draw).stop(); + + for (auto &cp : canvas_list) { + auto const ¶m = cp.first; + auto &[canvas, points] = cp.second; + log_gen("Handling param='{}'", param); + + if (!m_image_path.empty()) { + // Save input image for debugging + save_image_to_file(canvas, tile, m_image_path, param, "i", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store input image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, param, + "i", m_image_table); + } + + if (m_buffer_size > 0) { + log_gen("Generalize (buffer={} Mercator units)...", + m_buffer_size * tile.extent() / + static_cast(m_image_extent)); + timer(m_timer_simplify).start(); + canvas.open_close(m_buffer_size); + timer(m_timer_simplify).stop(); + } + + if (!m_image_path.empty()) { + // Save output image for debugging + save_image_to_file(canvas, tile, m_image_path, param, "o", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store output image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, param, + "o", m_image_table); + } + + tracer_t tracer{m_image_extent, m_image_buffer, m_turdsize}; + + log_gen("Vectorize..."); + timer(m_timer_vectorize).start(); + auto const geometries = tracer.trace(canvas, tile); + timer(m_timer_vectorize).stop(); + + log_gen("Reduced from {} points to {} points ({:.1f} %)", points, + tracer.num_points(), + static_cast(tracer.num_points()) / + static_cast(points) * 100); + + log_gen("Write geometries to destination table..."); + timer(m_timer_write).start(); + for (auto const &geom : geometries) { + auto const wkb = geom_to_ewkb(geom); + connection().exec_prepared("insert_geoms", binary_param{wkb}, + tile.x(), tile.y(), param); + } + timer(m_timer_write).stop(); + log_gen("Inserted {} generalized polygons", geometries.size()); + } +} + +void gen_tile_raster_union_t::post() +{ + if (!m_image_table.empty()) { + for (char const variant : {'i', 'o'}) { + raster_table_postprocess( + fmt::format("{}_{}", m_image_table, variant)); + } + } + dbexec("ANALYZE {dest}"); +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile-raster.hpp osm2pgsql-1.9.0+ds/src/gen/gen-tile-raster.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile-raster.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile-raster.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,49 @@ +#ifndef OSM2PGSQL_GEN_TILE_RASTER_HPP +#define OSM2PGSQL_GEN_TILE_RASTER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include +#include + +class gen_tile_raster_union_t final : public gen_tile_t +{ +public: + gen_tile_raster_union_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_raster_union_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override + { + return "raster-union"; + } + +private: + std::size_t m_timer_draw; + std::size_t m_timer_simplify; + std::size_t m_timer_vectorize; + std::size_t m_timer_write; + + std::string m_image_path; + std::string m_image_table; + double m_margin = 0.0; + std::size_t m_image_extent = 2048; + std::size_t m_image_buffer = 0; + unsigned int m_buffer_size = 10; + int m_turdsize = 2; +}; + +#endif // OSM2PGSQL_GEN_TILE_RASTER_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile-vector.cpp osm2pgsql-1.9.0+ds/src/gen/gen-tile-vector.cpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile-vector.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile-vector.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,96 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-vector.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +gen_tile_vector_union_t::gen_tile_vector_union_t(pg_conn_t *connection, + params_t *params) +: gen_tile_t(connection, params), m_timer_simplify(add_timer("simplify")) +{ + check_src_dest_table_params_exist(); + + if (!get_params().has("margin")) { + params->set("margin", 0.0); + } else { + // We don't need the result, just checking that this is a real number + get_params().get_double("margin"); + } + + if (!get_params().has("buffer_size")) { + params->set("buffer_size", static_cast(10)); + } else { + // We don't need the result, just checking that this is an integer + get_params().get_int64("buffer_size"); + } + + if (with_group_by()) { + dbexec(R"( +PREPARE gen_geoms (int, int, int) AS + WITH gen_tile_input AS ( + SELECT "{group_by_column}" AS col, "{geom_column}" AS geom FROM {src} + WHERE "{geom_column}" && ST_TileEnvelope($1, $2, $3, margin => {margin}) + ), + buffered AS ( + SELECT col, ST_Buffer(geom, {buffer_size}) AS geom + FROM gen_tile_input + ), + merged AS ( + SELECT col, ST_Union(geom) AS geom + FROM buffered GROUP BY col + ), + unbuffered AS ( + SELECT col, ST_Buffer(ST_Buffer(geom, -2 * {buffer_size}), {buffer_size}) AS geom + FROM merged + ) + INSERT INTO {dest} (x, y, "{group_by_column}", "{geom_column}") + SELECT $2, $3, col, (ST_Dump(geom)).geom FROM unbuffered +)"); + } else { + dbexec(R"( +PREPARE gen_geoms (int, int, int) AS + WITH gen_tile_input AS ( + SELECT "{geom_column}" AS geom FROM {src} + WHERE "{geom_column}" && ST_TileEnvelope($1, $2, $3, margin => {margin}) + ), + buffered AS ( + SELECT ST_Buffer(geom, {buffer_size}) AS geom + FROM gen_tile_input + ), + merged AS ( + SELECT ST_Union(geom) AS geom + FROM buffered + ), + unbuffered AS ( + SELECT ST_Buffer(ST_Buffer(geom, -2 * {buffer_size}), {buffer_size}) AS geom + FROM merged + ) + INSERT INTO {dest} (x, y, "{geom_column}") + SELECT $2, $3, (ST_Dump(geom)).geom FROM unbuffered +)"); + } +} + +void gen_tile_vector_union_t::process(tile_t const &tile) +{ + delete_existing(tile); + + log_gen("Generalize..."); + timer(m_timer_simplify).start(); + auto const result = connection().exec_prepared("gen_geoms", tile.zoom(), + tile.x(), tile.y()); + timer(m_timer_simplify).stop(); + log_gen("Inserted {} generalized polygons", result.affected_rows()); +} + +void gen_tile_vector_union_t::post() { dbexec("ANALYZE {dest}"); } diff -Nru osm2pgsql-1.8.1+ds/src/gen/gen-tile-vector.hpp osm2pgsql-1.9.0+ds/src/gen/gen-tile-vector.hpp --- osm2pgsql-1.8.1+ds/src/gen/gen-tile-vector.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/gen-tile-vector.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,37 @@ +#ifndef OSM2PGSQL_GEN_TILE_VECTOR_HPP +#define OSM2PGSQL_GEN_TILE_VECTOR_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include + +class gen_tile_vector_union_t final : public gen_tile_t +{ +public: + gen_tile_vector_union_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_vector_union_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override + { + return "vector-union"; + } + +private: + std::size_t m_timer_simplify; +}; + +#endif // OSM2PGSQL_GEN_TILE_VECTOR_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/osm2pgsql-gen.cpp osm2pgsql-1.9.0+ds/src/gen/osm2pgsql-gen.cpp --- osm2pgsql-1.8.1+ds/src/gen/osm2pgsql-gen.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/osm2pgsql-gen.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,761 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * This program is used for accessing generalization functionality. It is + * experimental and might or might not be integrated into osm2pgsql itself + * in the future. + */ + +#include "canvas.hpp" +#include "debug-output.hpp" +#include "expire-output.hpp" +#include "flex-lua-expire-output.hpp" +#include "flex-lua-geom.hpp" +#include "flex-lua-table.hpp" +#include "flex-table.hpp" +#include "format.hpp" +#include "gen-base.hpp" +#include "gen-create.hpp" +#include "logging.hpp" +#include "lua-init.hpp" +#include "lua-setup.hpp" +#include "lua-utils.hpp" +#include "options.hpp" +#include "params.hpp" +#include "pgsql-capabilities.hpp" +#include "pgsql.hpp" +#include "properties.hpp" +#include "tile.hpp" +#include "util.hpp" +#include "version.hpp" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr std::size_t const max_force_single_thread = 4; + +// Lua can't call functions on C++ objects directly. This macro defines simple +// C "trampoline" functions which are called from Lua which get the current +// context (the genproc_t object) and call the respective function on the +// context object. +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define TRAMPOLINE(func_name, lua_name) \ + static int lua_trampoline_##func_name(lua_State *lua_state) \ + { \ + try { \ + return static_cast(luaX_get_context(lua_state)) \ + ->func_name(); \ + } catch (std::exception const &e) { \ + return luaL_error(lua_state, "Error in '" #lua_name "': %s\n", \ + e.what()); \ + } catch (...) { \ + return luaL_error(lua_state, \ + "Unknown error in '" #lua_name "'.\n"); \ + } \ + } + +static void show_help() +{ + fmt::print(R"(osm2pgsql-gen [OPTIONS] +Generalization of OSM data. + +This program is EXPERIMENTAL and might change without notice. + +Main Options: + -a|--append Run in append mode + -c|--create Run in create mode (default) + -S|--style=FILE The Lua config file (same as for osm2pgsql) + -j|--jobs=NUM Number of parallel jobs (default 1) + --middle-schema=SCHEMA Database schema for middle tables (default set with --schema) + --schema=SCHEMA Default database schema (default: 'public') + +Help/Version Options: + -h|--help Print this help text and stop + -V|--version Show version + +Logging options: + -l|--log-level=LEVEL Log level (debug, info (default), warn, error) + --log-sql Log SQL commands + +Database options: + -d|--database=DB The name of the PostgreSQL database to connect to or + a PostgreSQL conninfo string. + -U|--username=NAME PostgreSQL user name. + -W|--password Force password prompt. + -H|--host=HOST Database server host name or socket location. + -P|--port=PORT Database server port. +)"); +} + +static char const *const short_options = "acd:hH:j:l:P:S:U:VW"; + +static std::array const long_options = { + {{"help", no_argument, nullptr, 'h'}, + {"version", no_argument, nullptr, 'V'}, + {"append", no_argument, nullptr, 'a'}, + {"create", no_argument, nullptr, 'c'}, + {"jobs", required_argument, nullptr, 'j'}, + {"database", required_argument, nullptr, 'd'}, + {"user", required_argument, nullptr, 'U'}, + {"host", required_argument, nullptr, 'H'}, + {"port", required_argument, nullptr, 'P'}, + {"password", no_argument, nullptr, 'W'}, + {"log-level", required_argument, nullptr, 'l'}, + {"style", required_argument, nullptr, 'S'}, + {"log-sql", no_argument, nullptr, 201}, + {"middle-schema", required_argument, nullptr, 202}, + {"schema", required_argument, nullptr, 203}, + {nullptr, 0, nullptr, 0}}}; + +struct tile_extent +{ + uint32_t xmin = 0; + uint32_t ymin = 0; + uint32_t xmax = 0; + uint32_t ymax = 0; + bool valid = false; +}; + +static bool table_is_empty(pg_conn_t const &db_connection, + std::string const &schema, std::string const &table) +{ + auto const result = db_connection.exec("SELECT 1 FROM {} LIMIT 1", + qualified_name(schema, table)); + return result.num_tuples() == 0; +} + +static tile_extent get_extent_from_db(pg_conn_t const &db_connection, + std::string const &schema, + std::string const &table, + std::string const &column, uint32_t zoom) +{ + if (table_is_empty(db_connection, schema, table)) { + return {}; + } + + auto const result = db_connection.exec( + "SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e)" + " FROM ST_EstimatedExtent('{}', '{}', '{}') AS e", + schema, table, column); + + if (result.num_tuples() == 0 || result.is_null(0, 0)) { + return {}; + } + + double const extent_xmin = strtod(result.get_value(0, 0), nullptr); + double const extent_ymin = strtod(result.get_value(0, 1), nullptr); + double const extent_xmax = strtod(result.get_value(0, 2), nullptr); + double const extent_ymax = strtod(result.get_value(0, 3), nullptr); + log_debug("Extent: ({} {}, {} {})", extent_xmin, extent_ymin, extent_xmax, + extent_ymax); + + return {osmium::geom::mercx_to_tilex(zoom, extent_xmin), + osmium::geom::mercy_to_tiley(zoom, extent_ymax), + osmium::geom::mercx_to_tilex(zoom, extent_xmax), + osmium::geom::mercy_to_tiley(zoom, extent_ymin), true}; +} + +static tile_extent get_extent_from_db(pg_conn_t const &db_connection, + std::string const &default_schema, + params_t const ¶ms, uint32_t zoom) +{ + auto const schema = params.get_string("schema", default_schema); + std::string table; + if (params.has("src_table")) { + table = params.get_string("src_table"); + } else if (params.has("src_tables")) { + table = params.get_string("src_tables"); + auto const n = table.find(','); + if (n != std::string::npos) { + table.resize(n); + } + } else { + throw std::runtime_error{"Need 'src_table' or 'src_tables' param."}; + } + auto const geom_column = params.get_string("geom_column", "geom"); + return get_extent_from_db(db_connection, schema, table, geom_column, zoom); +} + +static std::vector> +get_tiles_from_table(pg_conn_t const &connection, std::string const &table) +{ + std::vector> tiles; + + auto const result = connection.exec(R"(SELECT x, y FROM "{}")", table); + + for (int n = 0; n < result.num_tuples(); ++n) { + char *end = nullptr; + auto const x = std::strtoul(result.get_value(n, 0), &end, 10); + auto const y = std::strtoul(result.get_value(n, 1), &end, 10); + tiles.emplace_back(x, y); + } + + return tiles; +} + +class tile_processor_t +{ +public: + tile_processor_t(gen_base_t *generalizer, std::size_t num_tiles) + : m_generalizer(generalizer), m_num_tiles(num_tiles) + {} + + void operator()(tile_t const &tile) + { + log_debug("Processing tile {}/{}/{} ({} of {})...", tile.zoom(), + tile.x(), tile.y(), ++m_count, m_num_tiles); + m_generalizer->process(tile); + } + +private: + gen_base_t *m_generalizer; + std::size_t m_count = 0; + std::size_t m_num_tiles; +}; + +void run_tile_gen(std::string const &conninfo, gen_base_t *master_generalizer, + params_t params, uint32_t zoom, + std::vector> *queue, + std::mutex *mut, unsigned int n) +{ + logger::init_thread(n); + + log_debug("Started generalizer thread for '{}'.", + master_generalizer->strategy()); + pg_conn_t db_connection{conninfo}; + std::string const strategy{master_generalizer->strategy()}; + auto generalizer = create_generalizer(strategy, &db_connection, ¶ms); + + while (true) { + std::pair p; + { + std::lock_guard const guard{*mut}; + if (queue->empty()) { + master_generalizer->merge_timers(*generalizer); + break; + } + p = queue->back(); + queue->pop_back(); + } + + generalizer->process({zoom, p.first, p.second}); + } + log_debug("Shutting down generalizer thread."); +} + +class genproc_t +{ +public: + genproc_t(std::string const &filename, std::string conninfo, + std::string dbschema, bool append, bool updatable, + uint32_t jobs); + + int app_define_table() + { +#if 0 + if (m_calling_context != calling_context::main) { + throw std::runtime_error{ + "Database tables have to be defined in the" + " main Lua code, not in any of the callbacks."}; + } +#endif + + return setup_flex_table(m_lua_state.get(), &m_tables, &m_expire_outputs, + m_dbschema, true, m_append); + } + + int app_define_expire_output() + { + return setup_flex_expire_output(m_lua_state.get(), m_dbschema, + &m_expire_outputs); + } + + int app_run_gen() + { + log_debug("Running configured generalizer (run {})...", ++m_gen_run); + + if (lua_type(lua_state(), 1) != LUA_TSTRING) { + throw std::runtime_error{"Argument #1 to 'run_gen' must be a " + "string naming the strategy."}; + } + + std::string const strategy = lua_tostring(lua_state(), 1); + log_debug("Generalizer strategy '{}'", strategy); + + if (lua_type(lua_state(), 2) != LUA_TTABLE) { + throw std::runtime_error{"Argument #2 to 'run_gen' must be a " + "table with parameters."}; + } + + auto params = parse_params(); + + if (!params.has("schema")) { + params.set("schema", m_dbschema); + } + + write_to_debug_log(params, "Params (config):"); + + log_debug("Connecting to database..."); + pg_conn_t db_connection{m_conninfo}; + + log_debug("Creating generalizer..."); + auto generalizer = + create_generalizer(strategy, &db_connection, ¶ms); + + log_debug("Generalizer '{}' ({}) initialized.", generalizer->name(), + generalizer->strategy()); + + if (m_append) { + params.set("delete_existing", true); + } + + write_to_debug_log(params, "Params (after initialization):"); + + if (generalizer->on_tiles()) { + process_tiles(db_connection, params, generalizer.get()); + } else { + generalizer->process(); + } + + log_debug("Running generalizer postprocessing..."); + generalizer->post(); + + log_debug("Generalizer processing done."); + + log_debug("Timers:"); + for (auto const &timer : generalizer->timers()) { + log_debug(fmt::format( + " {:10} {:>10L}", timer.name() + ":", + std::chrono::duration_cast( + timer.elapsed()))); + } + log_debug("Finished generalizer '{}' (run {}).", generalizer->name(), + m_gen_run); + + return 0; + } + + int app_run_sql() + { + if (lua_type(lua_state(), 1) != LUA_TTABLE) { + throw std::runtime_error{"Argument #1 to 'run_sql' must be a " + "table with parameters."}; + } + + std::string const description = + luaX_get_table_string(lua_state(), "description", 1, "Argument #1"); + std::string const sql = + luaX_get_table_string(lua_state(), "sql", 1, "Argument #1"); + + log_debug("Running SQL command: {}.", description); + + util::timer_t timer_sql; + pg_conn_t const db_connection{m_conninfo}; + db_connection.exec(sql); + log_debug("SQL command took {}.", + util::human_readable_duration(timer_sql.stop())); + + return 0; + } + + void run(); + +private: + params_t parse_params() + { + params_t params; + + lua_pushnil(lua_state()); + while (lua_next(lua_state(), 2) != 0) { + if (lua_type(lua_state(), -2) != LUA_TSTRING) { + throw std::runtime_error{"Argument #2 must have string keys"}; + } + auto const *key = lua_tostring(lua_state(), -2); + + switch (lua_type(lua_state(), -1)) { + case LUA_TSTRING: + params.set(key, lua_tostring(lua_state(), -1)); + break; + case LUA_TNUMBER: +#if LUA_VERSION_NUM >= 503 + if (lua_isinteger(lua_state(), -1)) { + params.set(key, static_cast( + lua_tointeger(lua_state(), -1))); + } else { + params.set(key, static_cast( + lua_tonumber(lua_state(), -1))); + } +#else + { + auto const value = + static_cast(lua_tonumber(lua_state(), -1)); + if (std::floor(value) == value) { + params.set(key, static_cast(value)); + } else { + params.set(key, value); + } + } +#endif + break; + case LUA_TBOOLEAN: + params.set(key, + static_cast(lua_toboolean(lua_state(), -1))); + break; + case LUA_TNIL: + break; + default: + throw std::runtime_error{"Argument #2 must have string values"}; + } + + lua_pop(lua_state(), 1); + } + return params; + } + + void process_tiles(pg_conn_t const &db_connection, params_t const ¶ms, + gen_base_t *generalizer) + { + uint32_t const zoom = generalizer->get_zoom(); + std::vector> tile_list; + if (m_append) { + auto const table = params.get_string("expire_list"); + log_debug("Running generalizer for expire list from table '{}'...", + table); + tile_list = get_tiles_from_table(db_connection, table); + log_debug("Truncating table '{}'...", table); + db_connection.exec("TRUNCATE {}", table); + } else { + auto const extent = + get_extent_from_db(db_connection, m_dbschema, params, zoom); + + if (extent.valid) { + auto const num_tiles = (extent.xmax - extent.xmin + 1) * + (extent.ymax - extent.ymin + 1); + log_debug("Running generalizer for bounding box x{}-{}, y{}-{}" + " on zoom={}...", + extent.xmin, extent.xmax, extent.ymin, extent.ymax, + zoom); + tile_list.reserve(num_tiles); + for (unsigned x = extent.xmin; x <= extent.xmax; ++x) { + for (unsigned y = extent.ymin; y <= extent.ymax; ++y) { + tile_list.emplace_back(x, y); + } + } + } else { + log_debug("Source table empty, nothing to do."); + } + } + log_debug("Need to process {} tiles.", tile_list.size()); + if (m_jobs == 1 || tile_list.size() < max_force_single_thread) { + log_debug("Running in single-threaded mode."); + tile_processor_t tp{generalizer, tile_list.size()}; + while (!tile_list.empty()) { + auto [x, y] = tile_list.back(); + tp({zoom, x, y}); + tile_list.pop_back(); + } + } else { + log_debug("Running in multi-threaded mode."); + std::mutex mut; + std::vector threads; + for (unsigned int n = 1; + n <= std::min(m_jobs, static_cast(tile_list.size())); + ++n) { + threads.emplace_back(run_tile_gen, m_conninfo, generalizer, + params, zoom, &tile_list, &mut, n); + } + for (auto &t : threads) { + t.join(); + } + } + } + + lua_State *lua_state() const noexcept { return m_lua_state.get(); } + + std::shared_ptr m_lua_state{ + luaL_newstate(), [](lua_State *state) { lua_close(state); }}; + + std::vector m_tables; + std::vector m_expire_outputs; + + std::string m_conninfo; + std::string m_dbschema; + std::size_t m_gen_run = 0; + uint32_t m_jobs; + bool m_append; + bool m_updatable; +}; // class genproc_t + +TRAMPOLINE(app_define_table, define_table) +TRAMPOLINE(app_define_expire_output, define_expire_output) +TRAMPOLINE(app_run_gen, run_gen) +TRAMPOLINE(app_run_sql, run_sql) + +genproc_t::genproc_t(std::string const &filename, std::string conninfo, + std::string dbschema, bool append, bool updatable, + uint32_t jobs) +: m_conninfo(std::move(conninfo)), m_dbschema(std::move(dbschema)), + m_jobs(jobs), m_append(append), m_updatable(updatable) +{ + setup_lua_environment(lua_state(), filename, append); + + luaX_add_table_func(lua_state(), "define_table", + lua_trampoline_app_define_table); + luaX_add_table_func(lua_state(), "define_expire_output", + lua_trampoline_app_define_expire_output); + + luaX_add_table_func(lua_state(), "run_gen", lua_trampoline_app_run_gen); + luaX_add_table_func(lua_state(), "run_sql", lua_trampoline_app_run_sql); + + lua_getglobal(lua_state(), "osm2pgsql"); + if (luaL_newmetatable(lua_state(), osm2pgsql_expire_output_name) != 1) { + throw std::runtime_error{"Internal error: Lua newmetatable failed."}; + } + lua_pushvalue(lua_state(), -1); // Copy of new metatable + + // Add metatable as osm2pgsql.ExpireOutput so we can access it from Lua + lua_setfield(lua_state(), -3, "ExpireOutput"); + + // Clean up stack + lua_settop(lua_state(), 0); + + init_geometry_class(lua_state()); + + // Load compiled in init.lua + if (luaL_dostring(lua_state(), lua_init())) { + throw fmt_error("Internal error in Lua setup: {}.", + lua_tostring(lua_state(), -1)); + } + + // Load user config file + luaX_set_context(lua_state(), this); + if (luaL_dofile(lua_state(), filename.c_str())) { + throw fmt_error("Error loading lua config: {}.", + lua_tostring(lua_state(), -1)); + } + + write_expire_output_list_to_debug_log(m_expire_outputs); + write_table_list_to_debug_log(m_tables); +} + +void genproc_t::run() +{ + lua_getglobal(lua_state(), "osm2pgsql"); + lua_getfield(lua_state(), -1, "process_gen"); + + if (lua_isnil(lua_state(), -1)) { + log_warn("No function 'osm2pgsql.process_gen()'. Nothing to do."); + return; + } + + if (luaX_pcall(lua_state(), 0, 0)) { + throw fmt_error( + "Failed to execute Lua function 'osm2pgsql.process_gen': {}.", + lua_tostring(lua_state(), -1)); + } + + if (!m_append) { + pg_conn_t const db_connection{m_conninfo}; + for (auto const &table : m_tables) { + if (table.id_type() == flex_table_index_type::tile && + (table.always_build_id_index() || m_updatable)) { + log_info("Creating tile (x/y) index on table '{}'...", + table.name()); + auto const sql = + fmt::format("CREATE INDEX ON {} USING BTREE (x, y) {}", + table.full_name(), + tablespace_clause(table.index_tablespace())); + db_connection.exec(sql); + } + } + } +} + +int main(int argc, char *argv[]) +{ + try { + database_options_t database_options; + std::string dbschema{"public"}; + std::string middle_dbschema{}; + std::string log_level; + std::string style; + uint32_t jobs = 1; + bool pass_prompt = false; + bool append = false; + + int c = 0; + // NOLINTNEXTLINE(concurrency-mt-unsafe) + while (-1 != (c = getopt_long(argc, argv, short_options, + long_options.data(), nullptr))) { + switch (c) { + case 'h': // --help + show_help(); + return 0; + case 'a': // --append + append = true; + break; + case 'c': // --create + append = false; + break; + case 'j': // --jobs + jobs = + std::clamp(std::strtoul(optarg, nullptr, 10), 1UL, 256UL); + break; + case 'd': // --database + database_options.db = optarg; + break; + case 'U': // --username + database_options.username = optarg; + break; + case 'W': // --password + pass_prompt = true; + break; + case 'H': // --host + database_options.host = optarg; + break; + case 'P': // --port + database_options.port = optarg; + break; + case 'l': // --log-level + log_level = optarg; + break; + case 'S': // --style + style = optarg; + break; + case 'V': // --version + log_info("osm2pgsql-gen version {}", get_osm2pgsql_version()); + canvas_t::info(); + return 0; + case 201: // --log-sql + get_logger().enable_sql(); + break; + case 202: // --middle-schema + middle_dbschema = optarg; + if (middle_dbschema.empty()) { + log_error("Schema must not be empty"); + return 2; + } + check_identifier(middle_dbschema, "--middle-schema"); + break; + case 203: // --schema + dbschema = optarg; + if (dbschema.empty()) { + log_error("Schema must not be empty"); + return 2; + } + check_identifier(dbschema, "--schema"); + break; + default: + log_error("Unknown argument"); + return 2; + } + } + + if (log_level == "debug") { + get_logger().set_level(log_level::debug); + } else if (log_level == "info" || log_level.empty()) { + get_logger().set_level(log_level::info); + } else if (log_level == "warn") { + get_logger().set_level(log_level::warn); + } else if (log_level == "error") { + get_logger().set_level(log_level::error); + } else { + log_error("Unknown log level: {}. " + "Use 'debug', 'info', 'warn', or 'error'.", + log_level); + return 2; + } + + if (jobs < 1 || jobs > 32) { + log_error("The --jobs/-j parameter must be between 1 and 32."); + return 2; + } + + if (middle_dbschema.empty()) { + middle_dbschema = dbschema; + } + + util::timer_t timer_overall; + + log_info("osm2pgsql-gen version {}", get_osm2pgsql_version()); + log_warn("This is an EXPERIMENTAL extension to osm2pgsql."); + + if (append) { + log_debug("Running in append mode."); + } else { + log_debug("Running in create mode."); + } + + if (jobs == 1) { + log_debug("Running in single-threaded mode."); + } else { + log_debug( + "Running in multi-threaded mode with a maximum of {} threads.", + jobs); + } + + if (pass_prompt) { + database_options.password = util::get_password(); + } + auto const conninfo = build_conninfo(database_options); + + log_debug("Checking database capabilities..."); + { + pg_conn_t const db_connection{conninfo}; + init_database_capabilities(db_connection); + } + + properties_t properties{conninfo, middle_dbschema}; + properties.load(); + + if (style.empty()) { + style = properties.get_string("style", ""); + if (style.empty()) { + log_error("Need --style/-S option"); + return 2; + } + } + + bool const updatable = properties.get_bool("updatable", false); + genproc_t gen{style, conninfo, dbschema, append, updatable, jobs}; + gen.run(); + + osmium::MemoryUsage const mem; + log_info("Memory: {}MB current, {}MB peak", mem.current(), mem.peak()); + + log_info("osm2pgsql-gen took {} overall.", + util::human_readable_duration(timer_overall.stop())); + } catch (std::exception const &e) { + log_error("{}", e.what()); + return 1; + } catch (...) { + log_error("Unknown exception."); + return 1; + } + + return 0; +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/params.cpp osm2pgsql-1.9.0+ds/src/gen/params.cpp --- osm2pgsql-1.8.1+ds/src/gen/params.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/params.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,125 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "params.hpp" + +#include "format.hpp" +#include "logging.hpp" +#include "overloaded.hpp" +#include "pgsql.hpp" + +std::string to_string(param_value_t const &value) +{ + return std::visit( + overloaded{[](null_param_t) { return std::string{}; }, + [](std::string val) { return val; }, + [](auto const &val) { return fmt::to_string(val); }}, + value); +} + +param_value_t params_t::get(std::string const &key) const +{ + return m_map.at(key); +} + +bool params_t::has(std::string const &key) const noexcept +{ + return m_map.count(key) > 0; +} + +bool params_t::get_bool(std::string const &key, bool default_value) const +{ + return get_by_type(key, default_value); +} + +int64_t params_t::get_int64(std::string const &key, int64_t default_value) const +{ + return get_by_type(key, default_value); +} + +double params_t::get_double(std::string const &key, double default_value) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + return default_value; + } + + if (std::holds_alternative(it->second)) { + return std::get(it->second); + } + + if (std::holds_alternative(it->second)) { + return static_cast(std::get(it->second)); + } + + throw fmt_error("Invalid value '{}' for {}.", to_string(it->second), key); +} + +std::string params_t::get_string(std::string const &key) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + throw fmt_error("Missing parameter '{}' on generalizer.", key); + } + return to_string(it->second); +} + +std::string params_t::get_string(std::string const &key, + std::string const &default_value) const +{ + return get_by_type(key, default_value); +} + +std::string params_t::get_identifier(std::string const &key) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + return {}; + } + std::string result = to_string(it->second); + check_identifier(result, key.c_str()); + return result; +} + +void params_t::check_identifier_with_default(std::string const &key, + std::string default_value) +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + m_map.emplace(key, std::move(default_value)); + } else { + check_identifier(to_string(it->second), key.c_str()); + } +} + +unsigned int uint_in_range(params_t const ¶ms, std::string const &key, + unsigned int min, unsigned int max, + unsigned int default_value) +{ + int64_t const value = params.get_int64(key, default_value); + if (value < 0 || value > std::numeric_limits::max()) { + throw fmt_error("Invalid value '{}' for parameter '{}'.", value, key); + } + auto uvalue = static_cast(value); + if (uvalue < min || uvalue > max) { + throw fmt_error("Invalid value '{}' for parameter '{}'.", value, key); + } + return uvalue; +} + +void write_to_debug_log(params_t const ¶ms, char const *message) +{ + if (!get_logger().debug_enabled()) { + return; + } + log_debug(message); + for (auto const &[key, value] : params) { + log_debug(" {}={}", key, to_string(value)); + } +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/params.hpp osm2pgsql-1.9.0+ds/src/gen/params.hpp --- osm2pgsql-1.8.1+ds/src/gen/params.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/params.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,97 @@ +#ifndef OSM2PGSQL_PARAMS_HPP +#define OSM2PGSQL_PARAMS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "logging.hpp" + +#include +#include +#include +#include + +/// A "NULL" value for a parameter. Same as not set. +using null_param_t = std::monostate; + +/// A parameter value can have one of several types. +using param_value_t = + std::variant; + +/// Convert a parameter value into a string. +std::string to_string(param_value_t const &value); + +/** + * A collection of parameters. + */ +class params_t +{ +public: + template + void set(K &&key, V &&value) + { + m_map.insert_or_assign(std::forward(key), std::forward(value)); + } + + template + void remove(K &&key) + { + m_map.erase(std::forward(key)); + } + + bool has(std::string const &key) const noexcept; + + param_value_t get(std::string const &key) const; + + bool get_bool(std::string const &key, bool default_value = false) const; + + int64_t get_int64(std::string const &key, int64_t default_value = 0) const; + + double get_double(std::string const &key, double default_value = 0.0) const; + + std::string get_string(std::string const &key) const; + + std::string get_string(std::string const &key, + std::string const &default_value) const; + + std::string get_identifier(std::string const &key) const; + + void check_identifier_with_default(std::string const &key, + std::string default_value); + + auto begin() const noexcept { return m_map.begin(); } + + auto end() const noexcept { return m_map.end(); } + +private: + template + T get_by_type(std::string const &key, T default_value) const + { + auto const it = m_map.find(key); + if (it == m_map.end()) { + return default_value; + } + + if (!std::holds_alternative(it->second)) { + throw fmt_error("Invalid value '{}' for {}.", to_string(it->second), + key); + } + return std::get(it->second); + } + + std::map m_map; +}; // class params_t + +void write_to_debug_log(params_t const ¶ms, char const *message); + +unsigned int uint_in_range(params_t const ¶ms, std::string const &key, + unsigned int min, unsigned int max, + unsigned int default_value); + +#endif // OSM2PGSQL_PARAMS_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/raster.cpp osm2pgsql-1.9.0+ds/src/gen/raster.cpp --- osm2pgsql-1.8.1+ds/src/gen/raster.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/raster.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,66 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "raster.hpp" + +#include "canvas.hpp" +#include "format.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +#include + +template +void append(std::string *str, T value) +{ + str->append(reinterpret_cast(&value), sizeof(T)); +} + +void add_raster_header(std::string *wkb, wkb_raster_header const &data) +{ + append(wkb, data.endianness); + append(wkb, data.version); + append(wkb, data.nBands); + append(wkb, data.scaleX); + append(wkb, data.scaleY); + append(wkb, data.ipX); + append(wkb, data.ipY); + append(wkb, data.skewX); + append(wkb, data.skewY); + append(wkb, data.srid); + append(wkb, data.width); + append(wkb, data.height); +} + +void add_raster_band(std::string *wkb, wkb_raster_band const &data) +{ + append(wkb, data.bits); + append(wkb, data.nodata); +} + +void save_image_to_file(canvas_t const &canvas, tile_t const &tile, + std::string const &path, std::string const ¶m, + char const *variant, unsigned int image_extent, + double margin) +{ + std::string name{fmt::format("{}-{}-{}-{}{}{}.", path, tile.x(), tile.y(), + param, param.empty() ? "" : "-", variant)}; + + // write image file + canvas.save(name + "png"); + + // write world file + auto const pixel_size = tile.extent() / image_extent; + name += "wld"; + auto *file = std::fopen(name.c_str(), "w"); + fmt::print(file, "{0}\n0.0\n0.0\n-{0}\n{1}\n{2}\n", pixel_size, + tile.xmin(margin) + pixel_size / 2, + tile.ymax(margin) - pixel_size / 2); + (void)std::fclose(file); +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/raster.hpp osm2pgsql-1.9.0+ds/src/gen/raster.hpp --- osm2pgsql-1.8.1+ds/src/gen/raster.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/raster.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,64 @@ +#ifndef OSM2PGSQL_RASTER_HPP +#define OSM2PGSQL_RASTER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include +#include + +class canvas_t; +class pg_conn_t; +class tile_t; + +/** + * \file + * + * Helper functions for creating raster images in PostgreSQL/PostGIS. + * https://trac.osgeo.org/postgis/wiki/WKTRaster/RFC/RFC2_V0WKBFormat + */ + +struct wkb_raster_header +{ + uint8_t endianness = +#if __BYTE_ORDER == __LITTLE_ENDIAN + 1 // Little Endian +#else + 0 // Big Endian +#endif + ; + uint16_t version = 0; + uint16_t nBands = 0; + double scaleX = 0.0; + double scaleY = 0.0; + double ipX = 0.0; + double ipY = 0.0; + double skewX = 0.0; + double skewY = 0.0; + int32_t srid = 3857; + uint16_t width = 0; + uint16_t height = 0; +}; + +struct wkb_raster_band +{ + uint8_t bits = 0; + uint8_t nodata = 0; +}; + +void add_raster_header(std::string *wkb, wkb_raster_header const &data); + +void add_raster_band(std::string *wkb, wkb_raster_band const &data); + +void save_image_to_file(canvas_t const &canvas, tile_t const &tile, + std::string const &path, std::string const ¶m, + char const *variant, unsigned int image_extent, + double margin); + +#endif // OSM2PGSQL_RASTER_HPP diff -Nru osm2pgsql-1.8.1+ds/src/gen/tracer.cpp osm2pgsql-1.9.0+ds/src/gen/tracer.cpp --- osm2pgsql-1.8.1+ds/src/gen/tracer.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/tracer.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,108 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "tracer.hpp" + +#include "geom-boost-adaptor.hpp" + +#include +#include +#include + +geom::point_t tracer_t::make_point(potrace_dpoint_t const &p) const noexcept +{ + return {p.x - static_cast(m_buffer), + static_cast(m_extent + m_buffer) - p.y}; +} + +std::vector +tracer_t::trace(canvas_t const &canvas, tile_t const &tile, double min_area) +{ + prepare(canvas); + + m_state.reset(potrace_trace(m_param.get(), &m_bitmap)); + if (!m_state || m_state->status != POTRACE_STATUS_OK) { + throw std::runtime_error{"potrace failed"}; + } + + return build_geometries(tile, m_state->plist, min_area); +} + +void tracer_t::reset() +{ + m_bits.clear(); + m_state.reset(); + m_num_points = 0; +} + +void tracer_t::prepare(canvas_t const &canvas) noexcept +{ + std::size_t const size = canvas.size(); + assert(size % bits_per_word == 0); + + m_bits.reserve((size * size) / bits_per_word); + + unsigned char const *d = canvas.begin(); + while (d != canvas.end()) { + potrace_word w = 0x1U & *d++; + for (std::size_t n = 1; n < bits_per_word; ++n) { + w <<= 1U; + assert(d != canvas.end()); + w |= 0x1U & *d++; + } + m_bits.push_back(w); + } + + m_bitmap = {int(size), int(size), int(size / bits_per_word), m_bits.data()}; +} + +std::vector +tracer_t::build_geometries(tile_t const &tile, potrace_path_t const *plist, + double min_area) noexcept +{ + std::vector geometries; + if (!plist) { + return geometries; + } + + for (potrace_path_t const *path = plist; path != nullptr; + path = path->next) { + + geom::ring_t ring; + + auto const n = path->curve.n; + assert(path->curve.tag[n - 1] == POTRACE_CORNER); + ring.push_back(tile.to_world_coords(make_point(path->curve.c[n - 1][2]), + m_extent)); + for (int i = 0; i < n; ++i) { + assert(path->curve.tag[i] == POTRACE_CORNER); + auto const &c = path->curve.c[i]; + ring.push_back(tile.to_world_coords(make_point(c[1]), m_extent)); + ring.push_back(tile.to_world_coords(make_point(c[2]), m_extent)); + } + + auto const ring_area = + std::abs(static_cast(boost::geometry::area(ring))); + if (ring_area >= min_area) { + m_num_points += ring.size(); + + if (path->sign == '+') { + geometries.emplace_back(geom::polygon_t{}, 3857) + .get() + .outer() = std::move(ring); + } else { + assert(!geometries.empty()); + geometries.back().get().add_inner_ring( + std::move(ring)); + } + } + } + + return geometries; +} diff -Nru osm2pgsql-1.8.1+ds/src/gen/tracer.hpp osm2pgsql-1.9.0+ds/src/gen/tracer.hpp --- osm2pgsql-1.8.1+ds/src/gen/tracer.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/gen/tracer.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,76 @@ +#ifndef OSM2PGSQL_TRACER_HPP +#define OSM2PGSQL_TRACER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "canvas.hpp" +#include "geom.hpp" +#include "tile.hpp" + +#include + +#include +#include + +class tracer_t +{ +public: + tracer_t(std::size_t extent, std::size_t buffer, int turdsize) + : m_param(potrace_param_default()), m_extent(extent), m_buffer(buffer) + { + m_param->alphamax = 0.0; + m_param->turdsize = turdsize; + } + + std::vector + trace(canvas_t const &canvas, tile_t const &tile, double min_area = 0.0); + + void reset(); + + std::size_t num_points() const noexcept { return m_num_points; } + +private: + static constexpr auto const bits_per_word = sizeof(potrace_word) * 8; + + geom::point_t make_point(potrace_dpoint_t const &p) const noexcept; + + struct potrace_param_deleter + { + void operator()(potrace_param_t *ptr) const noexcept + { + potrace_param_free(ptr); + } + }; + + struct potrace_state_deleter + { + void operator()(potrace_state_t *ptr) const noexcept + { + potrace_state_free(ptr); + } + }; + + void prepare(canvas_t const &canvas) noexcept; + + std::vector build_geometries(tile_t const &tile, + potrace_path_t const *plist, + double min_area) noexcept; + + std::vector m_bits; + potrace_bitmap_t m_bitmap{}; + std::unique_ptr m_param; + std::unique_ptr m_state; + std::size_t m_extent; + std::size_t m_buffer; + std::size_t m_num_points = 0; + +}; // class tracer_t + +#endif // OSM2PGSQL_TRACER_HPP diff -Nru osm2pgsql-1.8.1+ds/src/geom-functions.cpp osm2pgsql-1.9.0+ds/src/geom-functions.cpp --- osm2pgsql-1.8.1+ds/src/geom-functions.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/geom-functions.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -8,7 +8,9 @@ */ #include "geom-functions.hpp" + #include "geom-boost-adaptor.hpp" +#include "overloaded.hpp" #include #include @@ -361,6 +363,44 @@ } /****************************************************************************/ + +static double spherical_area(polygon_t const &geom) +{ + boost::geometry::strategy::area::spherical<> const spherical_earth{ + 6371008.8}; + + using sph_point = boost::geometry::model::point< + double, 2, + boost::geometry::cs::spherical_equatorial>; + + boost::geometry::model::polygon sph_geom; + boost::geometry::convert(geom, sph_geom); + return boost::geometry::area(sph_geom, spherical_earth); +} + +double spherical_area(geometry_t const &geom) +{ + assert(geom.srid() == 4326); + + return std::abs(geom.visit(overloaded{ + [&](geom::nullgeom_t const & /*input*/) { return 0.0; }, + [&](geom::collection_t const &input) { + return std::accumulate(input.cbegin(), input.cend(), 0.0, + [](double sum, auto const &geom) { + return sum + spherical_area(geom); + }); + }, + [&](geom::polygon_t const &input) { return spherical_area(input); }, + [&](geom::multipolygon_t const &input) { + return std::accumulate(input.cbegin(), input.cend(), 0.0, + [&](double sum, auto const &geom) { + return sum + spherical_area(geom); + }); + }, + [&](auto const & /*input*/) { return 0.0; }})); +} + +/****************************************************************************/ double length(geometry_t const &geom) { diff -Nru osm2pgsql-1.8.1+ds/src/geom-functions.hpp osm2pgsql-1.9.0+ds/src/geom-functions.hpp --- osm2pgsql-1.8.1+ds/src/geom-functions.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/geom-functions.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -148,6 +148,17 @@ double area(geometry_t const &geom); /** + * Calculate area of geometry on the spheroid. + * For geometry types other than polygon or multipolygon this will always + * return 0. + * + * \param geom Input geometry. + * \returns Area in m². + * \pre \code geom.srid() == 4326 \endcode + */ +double spherical_area(geometry_t const &geom); + +/** * Split multigeometries into their parts. Non-multi geometries are left * alone and will end up as the only geometry in the result vector. If the * input geometry is a nullgeom_t, the result vector will be empty. diff -Nru osm2pgsql-1.8.1+ds/src/geom.hpp osm2pgsql-1.9.0+ds/src/geom.hpp --- osm2pgsql-1.8.1+ds/src/geom.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/geom.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -84,6 +84,23 @@ return !(a == b); } + /// Give points an (arbitrary) order. + [[nodiscard]] constexpr friend bool operator<(point_t a, point_t b) noexcept + { + if (a.x() == b.x()) { + return a.y() < b.y(); + } + return a.x() < b.x(); + } + + [[nodiscard]] constexpr friend bool operator>(point_t a, point_t b) noexcept + { + if (a.x() == b.x()) { + return a.y() > b.y(); + } + return a.x() > b.x(); + } + private: double m_x = 0.0; double m_y = 0.0; @@ -403,16 +420,4 @@ } // namespace geom -// This magic is used for visiting geometries. For an explanation see for -// instance here: -// https://arne-mertz.de/2018/05/overload-build-a-variant-visitor-on-the-fly/ -template -struct overloaded : Ts... -{ - using Ts::operator()...; -}; - -template -overloaded(Ts...) -> overloaded; - #endif // OSM2PGSQL_GEOM_HPP diff -Nru osm2pgsql-1.8.1+ds/src/geom-transform.hpp osm2pgsql-1.9.0+ds/src/geom-transform.hpp --- osm2pgsql-1.8.1+ds/src/geom-transform.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/geom-transform.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -17,10 +17,7 @@ #include #include -extern "C" -{ -#include -} +#include #include diff -Nru osm2pgsql-1.8.1+ds/src/input.cpp osm2pgsql-1.9.0+ds/src/input.cpp --- osm2pgsql-1.8.1+ds/src/input.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/input.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -261,11 +261,14 @@ bool m_append; }; // class input_context_t -static void process_single_file(osmium::io::File const &file, - osmdata_t *osmdata, - progress_display_t *progress, bool append) +static file_info process_single_file(osmium::io::File const &file, + osmdata_t *osmdata, + progress_display_t *progress, bool append) { + file_info finfo; + osmium::io::Reader reader{file}; + finfo.header = reader.header(); type_id last{osmium::item_type::node, 0}; input_context_t ctx{osmdata, progress, append}; @@ -273,17 +276,25 @@ for (auto &object : buffer.select()) { last = check_input(last, object); ctx.apply(&object); + if (object.timestamp() > finfo.last_timestamp) { + finfo.last_timestamp = object.timestamp(); + } } } ctx.eof(); reader.close(); + + return finfo; } -static void process_multiple_files(std::vector const &files, - osmdata_t *osmdata, - progress_display_t *progress, bool append) +static file_info +process_multiple_files(std::vector const &files, + osmdata_t *osmdata, progress_display_t *progress, + bool append) { + file_info finfo; + std::vector data_sources; data_sources.reserve(files.size()); @@ -303,6 +314,9 @@ queue.pop(); if (queue.empty() || element != queue.top()) { ctx.apply(&element.object()); + if (element.object().timestamp() > finfo.last_timestamp) { + finfo.last_timestamp = element.object().timestamp(); + } } auto *source = element.data_source(); @@ -315,18 +329,20 @@ for (auto &data_source : data_sources) { data_source.close(); } + + return finfo; } -void process_files(std::vector const &files, - osmdata_t *osmdata, bool append, bool show_progress) +file_info process_files(std::vector const &files, + osmdata_t *osmdata, bool append, bool show_progress) { assert(osmdata); progress_display_t progress{show_progress}; if (files.size() == 1) { - process_single_file(files.front(), osmdata, &progress, append); - } else { - process_multiple_files(files, osmdata, &progress, append); + return process_single_file(files.front(), osmdata, &progress, append); } + + return process_multiple_files(files, osmdata, &progress, append); } diff -Nru osm2pgsql-1.8.1+ds/src/input.hpp osm2pgsql-1.9.0+ds/src/input.hpp --- osm2pgsql-1.8.1+ds/src/input.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/input.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -13,8 +13,6 @@ /** * \file * - * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql). - * * It contains the functions reading and checking the input data. */ @@ -23,6 +21,7 @@ #include #include +#include #include "osmtypes.hpp" @@ -34,6 +33,12 @@ osmid_t id; }; +struct file_info +{ + osmium::io::Header header{}; + osmium::Timestamp last_timestamp{}; +}; + /** * Compare two tuples (type, id). Throw a descriptive error if either the * curr id is negative or if the data is not ordered. @@ -53,7 +58,7 @@ /** * Process the specified OSM files (stage 1a). */ -void process_files(std::vector const &files, - osmdata_t *osmdata, bool append, bool show_progress); +file_info process_files(std::vector const &files, + osmdata_t *osmdata, bool append, bool show_progress); #endif // OSM2PGSQL_INPUT_HPP diff -Nru osm2pgsql-1.8.1+ds/src/logging.cpp osm2pgsql-1.9.0+ds/src/logging.cpp --- osm2pgsql-1.8.1+ds/src/logging.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/logging.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -9,6 +9,8 @@ #include "logging.hpp" +#include + thread_local unsigned int this_thread_num = 0; /// Global logger singleton @@ -17,26 +19,29 @@ /// Access the global logger singleton logger &get_logger() noexcept { return the_logger; } -std::string logger::generate_common_prefix(fmt::text_style const &ts, - char const *prefix) +void logger::generate_common_prefix(std::string *str, fmt::text_style const &ts, + char const *prefix) const { - std::string str; - - if (m_needs_leading_return) { - m_needs_leading_return = false; - str += '\n'; - } - - str += fmt::format("{:%Y-%m-%d %H:%M:%S} ", + *str += fmt::format("{:%Y-%m-%d %H:%M:%S} ", fmt::localtime(std::time(nullptr))); if (m_current_level == log_level::debug) { - str += fmt::format(ts, "[{}] ", this_thread_num); + *str += fmt::format(ts, "[{:02d}] ", this_thread_num); } if (prefix) { - str += fmt::format(ts, "{}: ", prefix); + *str += fmt::format(ts, "{}: ", prefix); } +} + +void logger::init_thread(unsigned int num) +{ + // Store thread number in thread local variable + this_thread_num = num; - return str; + // Set thread name in operating system. + // On Linux thread names have a maximum length of 16 characters. + std::string name{"_osm2pgsql_"}; + name.append(std::to_string(num)); + osmium::thread::set_thread_name(name.c_str()); } diff -Nru osm2pgsql-1.8.1+ds/src/logging.hpp osm2pgsql-1.9.0+ds/src/logging.hpp --- osm2pgsql-1.8.1+ds/src/logging.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/logging.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -17,11 +17,10 @@ #include #include +#include #include #include -extern thread_local unsigned int this_thread_num; - enum class log_level { debug = 1, @@ -36,9 +35,6 @@ */ class logger { - std::string generate_common_prefix(fmt::text_style const &ts, - char const *prefix); - public: template void log(log_level with_level, char const *prefix, @@ -50,7 +46,14 @@ auto const &ts = m_use_color ? style : fmt::text_style{}; - auto str = generate_common_prefix(ts, prefix); + std::string str; + + if (m_needs_leading_return) { + m_needs_leading_return = false; + str += '\n'; + } + + generate_common_prefix(&str, ts, prefix); str += fmt::format(ts, format_str, std::forward(args)...); str += '\n'; @@ -66,6 +69,11 @@ void set_level(log_level level) noexcept { m_current_level = level; } + bool debug_enabled() const noexcept + { + return m_current_level == log_level::debug; + } + void enable_sql() noexcept { m_log_sql = true; } void enable_sql_data() noexcept { m_log_sql_data = true; } @@ -79,12 +87,17 @@ void needs_leading_return() noexcept { m_needs_leading_return = true; } void no_leading_return() noexcept { m_needs_leading_return = false; } + static void init_thread(unsigned int num); + private: + void generate_common_prefix(std::string *str, fmt::text_style const &ts, + char const *prefix) const; + log_level m_current_level = log_level::info; bool m_log_sql = false; bool m_log_sql_data = false; bool m_show_progress = true; - bool m_needs_leading_return = false; + std::atomic m_needs_leading_return = false; #ifdef _WIN32 bool m_use_color = false; diff -Nru osm2pgsql-1.8.1+ds/src/lua-setup.cpp osm2pgsql-1.9.0+ds/src/lua-setup.cpp --- osm2pgsql-1.8.1+ds/src/lua-setup.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/lua-setup.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,39 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "lua-setup.hpp" +#include "lua-utils.hpp" +#include "version.hpp" + +#include + +#include + +void setup_lua_environment(lua_State *lua_state, std::string const &filename, + bool append_mode) +{ + // Set up global lua libs + luaL_openlibs(lua_state); + + // Set up global "osm2pgsql" object + lua_newtable(lua_state); + lua_pushvalue(lua_state, -1); + lua_setglobal(lua_state, "osm2pgsql"); + + luaX_add_table_str(lua_state, "version", get_osm2pgsql_short_version()); + + std::string dir_path = + boost::filesystem::path{filename}.parent_path().string(); + if (!dir_path.empty()) { + dir_path += boost::filesystem::path::preferred_separator; + } + luaX_add_table_str(lua_state, "config_dir", dir_path.c_str()); + + luaX_add_table_str(lua_state, "mode", append_mode ? "append" : "create"); +} diff -Nru osm2pgsql-1.8.1+ds/src/lua-setup.hpp osm2pgsql-1.9.0+ds/src/lua-setup.hpp --- osm2pgsql-1.8.1+ds/src/lua-setup.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/lua-setup.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,20 @@ +#ifndef OSM2PGSQL_LUA_CONFIG_HPP +#define OSM2PGSQL_LUA_CONFIG_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include + +struct lua_State; + +void setup_lua_environment(lua_State *lua_state, std::string const &filename, + bool append_mode); + +#endif // OSM2PGSQL_LUA_CONFIG_HPP diff -Nru osm2pgsql-1.8.1+ds/src/lua-utils.cpp osm2pgsql-1.9.0+ds/src/lua-utils.cpp --- osm2pgsql-1.8.1+ds/src/lua-utils.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/lua-utils.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -8,12 +8,8 @@ */ #include "lua-utils.hpp" -#include "format.hpp" -extern "C" -{ -#include -} +#include "format.hpp" #include #include @@ -136,8 +132,9 @@ return default_value; } if (ltype != LUA_TSTRING) { - throw fmt_error("{} field '{}' must be a string field.", error_msg, - key); + throw fmt_error("{} field must contain a '{}' string field " + "(or nil for default: '{}').", + error_msg, key, default_value); } return lua_tostring(lua_state, -1); } @@ -162,6 +159,21 @@ throw fmt_error("{} field '{}' must be a boolean field.", error_msg, key); } +uint32_t luaX_get_table_optional_uint32(lua_State *lua_state, char const *key, + int table_index, char const *error_msg) +{ + assert(lua_state); + assert(key); + assert(error_msg); + lua_getfield(lua_state, table_index, key); + if (lua_isnil(lua_state, -1)) { + return 0; + } + if (!lua_isnumber(lua_state, -1)) { + throw fmt_error("{} must contain an integer.", error_msg); + } + return lua_tonumber(lua_state, -1); +} // Lua 5.1 doesn't support luaL_traceback, unless LuaJIT is used #if LUA_VERSION_NUM < 502 && !defined(HAVE_LUAJIT) diff -Nru osm2pgsql-1.8.1+ds/src/lua-utils.hpp osm2pgsql-1.9.0+ds/src/lua-utils.hpp --- osm2pgsql-1.8.1+ds/src/lua-utils.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/lua-utils.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -13,10 +13,7 @@ // This file contains helper functions for talking to Lua. It is used from // the flex output backend. All functions start with "luaX_". -extern "C" -{ -#include -} +#include #include #include @@ -60,6 +57,9 @@ int table_index, char const *error_msg, char const *default_value); +uint32_t luaX_get_table_optional_uint32(lua_State *lua_state, char const *key, + int table_index, char const *error_msg); + bool luaX_get_table_bool(lua_State *lua_state, char const *key, int table_index, char const *error_msg, bool default_value); diff -Nru osm2pgsql-1.8.1+ds/src/middle.hpp osm2pgsql-1.9.0+ds/src/middle.hpp --- osm2pgsql-1.8.1+ds/src/middle.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/middle.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,6 +10,7 @@ * For a full list of authors see the git log. */ +#include #include #include @@ -18,7 +19,7 @@ #include "osmtypes.hpp" #include "thread-pool.hpp" -class options_t; +struct options_t; struct output_requirements; /** @@ -26,8 +27,15 @@ */ struct middle_query_t : std::enable_shared_from_this { + middle_query_t() noexcept = default; + virtual ~middle_query_t() = 0; + middle_query_t(middle_query_t const &) = delete; + middle_query_t &operator=(middle_query_t const &) = delete; + middle_query_t(middle_query_t &&) = delete; + middle_query_t &operator=(middle_query_t &&) = delete; + /** * Retrieves node location for the given id. */ @@ -95,6 +103,11 @@ virtual ~middle_t() = 0; + middle_t(middle_t const &) = delete; + middle_t &operator=(middle_t const &) = delete; + middle_t(middle_t &&) = delete; + middle_t &operator=(middle_t &&) = delete; + virtual void start() = 0; virtual void stop() = 0; @@ -110,17 +123,44 @@ virtual void relation(osmium::Relation const &relation) = 0; /// Called after all nodes from the input file(s) have been processed. - virtual void after_nodes() {} + virtual void after_nodes() + { + assert(m_middle_state == middle_state::node); +#ifndef NDEBUG + m_middle_state = middle_state::way; +#endif + } /// Called after all ways from the input file(s) have been processed. - virtual void after_ways() {} + virtual void after_ways() + { + assert(m_middle_state == middle_state::way); +#ifndef NDEBUG + m_middle_state = middle_state::relation; +#endif + } /// Called after all relations from the input file(s) have been processed. - virtual void after_relations() {} + virtual void after_relations() + { + assert(m_middle_state == middle_state::relation); +#ifndef NDEBUG + m_middle_state = middle_state::done; +#endif + } + + virtual void get_node_parents( + osmium::index::IdSetSmall const & /*changed_nodes*/, + osmium::index::IdSetSmall * /*parent_ways*/, + osmium::index::IdSetSmall * /*parent_relations*/) const + { + } - virtual idlist_t get_ways_by_node(osmid_t) { return {}; } - virtual idlist_t get_rels_by_node(osmid_t) { return {}; } - virtual idlist_t get_rels_by_way(osmid_t) { return {}; } + virtual void get_way_parents( + osmium::index::IdSetSmall const & /*changed_ways*/, + osmium::index::IdSetSmall * /*parent_relations*/) const + { + } virtual std::shared_ptr get_query_instance() = 0; @@ -133,6 +173,21 @@ return *m_thread_pool; } +#ifndef NDEBUG + enum class middle_state + { + constructed, + started, + node, + way, + relation, + done + }; + + // NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes, misc-non-private-member-variables-in-classes) + middle_state m_middle_state = middle_state::constructed; +#endif + private: std::shared_ptr m_thread_pool; }; // class middle_t diff -Nru osm2pgsql-1.8.1+ds/src/middle-pgsql.cpp osm2pgsql-1.9.0+ds/src/middle-pgsql.cpp --- osm2pgsql-1.8.1+ds/src/middle-pgsql.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/middle-pgsql.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -15,8 +15,11 @@ * emit the final geometry-enabled output formats */ +#include #include +#include #include +#include #include #include #include @@ -27,7 +30,10 @@ #include #include +#include + #include "format.hpp" +#include "json-writer.hpp" #include "logging.hpp" #include "middle-pgsql.hpp" #include "node-locations.hpp" @@ -37,37 +43,99 @@ #include "pgsql-helper.hpp" #include "util.hpp" -static std::string build_sql(options_t const &options, char const *templ) +static bool check_bucket_index(pg_conn_t const *db_connection, + std::string const &prefix) +{ + auto const res = + db_connection->exec("SELECT relname FROM pg_class" + " WHERE relkind='i'" + " AND relname = '{}_ways_nodes_bucket_idx'", + prefix); + return res.num_tuples() > 0; +} + +static void send_id_list(pg_conn_t const &db_connection, + std::string const &table, + osmium::index::IdSetSmall const &ids) +{ + std::string data; + for (auto const id : ids) { + fmt::format_to(std::back_inserter(data), FMT_STRING("{}\n"), id); + } + + auto const sql = fmt::format("COPY {} FROM STDIN", table); + db_connection.copy_start(sql.c_str()); + db_connection.copy_send(data, table); + db_connection.copy_end(table); +} + +static void load_id_list(pg_conn_t const &db_connection, + std::string const &table, + osmium::index::IdSetSmall *ids) +{ + auto const res = + db_connection.exec(fmt::format("SELECT id FROM {} ORDER BY id", table)); + for (int n = 0; n < res.num_tuples(); ++n) { + ids->set(osmium::string_to_object_id(res.get_value(n, 0))); + } +} + +static std::string build_sql(options_t const &options, std::string const &templ) { std::string const using_tablespace{options.tblsslim_index.empty() ? "" : "USING INDEX TABLESPACE " + options.tblsslim_index}; + + std::string const schema = "\"" + options.middle_dbschema + "\"."; + return fmt::format( - templ, fmt::arg("prefix", options.prefix), - fmt::arg("schema", options.middle_dbschema.empty() - ? "" - : ("\"" + options.middle_dbschema + "\".")), + fmt::runtime(templ), fmt::arg("prefix", options.prefix), + fmt::arg("schema", schema), fmt::arg("unlogged", options.droptemp ? "UNLOGGED" : ""), fmt::arg("using_tablespace", using_tablespace), fmt::arg("data_tablespace", tablespace_clause(options.tblsslim_data)), fmt::arg("index_tablespace", tablespace_clause(options.tblsslim_index)), - fmt::arg("way_node_index_id_shift", options.way_node_index_id_shift)); + fmt::arg("way_node_index_id_shift", options.way_node_index_id_shift), + fmt::arg("attribute_columns_definition", + options.extra_attributes ? " created timestamp with time zone," + " version int4," + " changeset_id int4," + " user_id int4," + : ""), + fmt::arg("attribute_columns_use", + options.extra_attributes + ? ", EXTRACT(EPOCH FROM created) AS created, version, " + "changeset_id, user_id, u.name" + : ""), + fmt::arg("users_table_access", + options.extra_attributes + ? "LEFT JOIN " + schema + '"' + options.prefix + + "_users\" u ON o.user_id = u.id" + : "")); +} + +static std::vector +build_sql(options_t const &options, std::vector const &templs) +{ + std::vector out; + out.reserve(templs.size()); + + for (auto const &templ : templs) { + out.push_back(build_sql(options, templ)); + } + + return out; } middle_pgsql_t::table_desc::table_desc(options_t const &options, table_sql const &ts) : m_create_table(build_sql(options, ts.create_table)), - m_prepare_query(build_sql(options, ts.prepare_query)), - m_copy_target(std::make_shared()) + m_prepare_queries(build_sql(options, ts.prepare_queries)), + m_copy_target(std::make_shared( + options.middle_dbschema, build_sql(options, ts.name), "id")) { - m_copy_target->name = build_sql(options, ts.name); - m_copy_target->schema = options.middle_dbschema; - m_copy_target->id = "id"; // XXX hardcoded column name - if (options.with_forward_dependencies) { - m_prepare_fw_dep_lookups = - build_sql(options, ts.prepare_fw_dep_lookups); m_create_fw_dep_indexes = build_sql(options, ts.create_fw_dep_indexes); } } @@ -102,7 +170,21 @@ pg_conn_t const db_connection{conninfo}; log_info("Building index on table '{}'", name()); - db_connection.exec(m_create_fw_dep_indexes); + for (auto const &query : m_create_fw_dep_indexes) { + db_connection.exec(query); + } +} + +void middle_pgsql_t::table_desc::init_max_id(pg_conn_t const &db_connection) +{ + auto const qual_name = qualified_name(schema(), name()); + auto const res = db_connection.exec("SELECT max(id) FROM {}", qual_name); + + if (res.is_null(0, 0)) { + return; + } + + m_max_id = osmium::string_to_object_id(res.get_value(0, 0)); } /** @@ -155,6 +237,54 @@ } namespace { + +/** + * Go through tags returned from a version 1 middle table, get the attributes + * from the "osm_*" tags and add them to the builder. + */ +template +void pgsql_get_attr_from_tags(char const *string, T *builder) +{ + if (*string++ != '{') { + return; + } + + std::string key; + std::string val; + std::string user; + while (*string != '}') { + string = decode_to_delimiter(string, &key); + // String points to the comma */ + ++string; + string = decode_to_delimiter(string, &val); + + if (key == "osm_version") { + builder->set_version(val.c_str()); + } else if (key == "osm_timestamp") { + builder->set_timestamp(osmium::Timestamp{val}); + } else if (key == "osm_changeset") { + builder->set_changeset(val.c_str()); + } else if (key == "osm_uid") { + builder->set_uid(val.c_str()); + } else if (key == "osm_user") { + user = val; + } + + // String points to the comma or closing '}' */ + if (*string == ',') { + ++string; + } + } + + // Must be done at the end, because after that call the builder might + // become invalid due to buffer resizing. + builder->set_user(user); +} + +/** + * Go through tags returned from a version 1 middle table, get all tags except + * the pseudo-tags containing attributes and add them to the builder. + */ template void pgsql_parse_tags(char const *string, osmium::memory::Buffer *buffer, T *obuilder) @@ -172,7 +302,12 @@ // String points to the comma */ ++string; string = decode_to_delimiter(string, &val); - builder.add_tag(key, val); + + if (key != "osm_version" && key != "osm_timestamp" && + key != "osm_changeset" && key != "osm_uid" && key != "osm_user") { + builder.add_tag(key, val); + } + // String points to the comma or closing '}' */ if (*string == ',') { ++string; @@ -180,6 +315,135 @@ } } +/** + * Parse JSON-encoded tags from a version 2 middle table and add them to the + * builder. + */ +template +void pgsql_parse_json_tags(char const *string, osmium::memory::Buffer *buffer, + T *obuilder) +{ + auto const tags = nlohmann::json::parse(string); + if (!tags.is_object()) { + throw std::runtime_error{"Database format for tags invalid."}; + } + + // These will come out sorted, because internally "tags" uses a std::map. + osmium::builder::TagListBuilder builder{*buffer, obuilder}; + for (auto const &tag : tags.items()) { + builder.add_tag(tag.key(), tag.value()); + } +} + +/** + * Helper class for parsing relation members encoded in JSON. + */ +class member_list_json_builder +{ +public: + explicit member_list_json_builder( + osmium::builder::RelationMemberListBuilder *builder) + : m_builder(builder) + {} + + static bool number_integer(nlohmann::json::number_integer_t /*val*/) + { + return true; + } + + bool number_unsigned(nlohmann::json::number_unsigned_t val) + { + m_ref = static_cast(val); + return true; + } + + static bool number_float(nlohmann::json::number_float_t /*val*/, + const nlohmann::json::string_t & /*s*/) + { + return true; + } + + bool key(nlohmann::json::string_t &val) + { + if (val == "type") { + m_next_val = next_val::type; + } else if (val == "ref") { + m_next_val = next_val::ref; + } else if (val == "role") { + m_next_val = next_val::role; + } else { + m_next_val = next_val::none; + } + return true; + } + + bool string(nlohmann::json::string_t &val) + { + if (m_next_val == next_val::type && val.size() == 1) { + switch (val[0]) { + case 'N': + m_type = osmium::item_type::node; + break; + case 'W': + m_type = osmium::item_type::way; + break; + default: + m_type = osmium::item_type::relation; + break; + } + } else if (m_next_val == next_val::role) { + m_role = val; + } + return true; + } + + bool end_object() + { + m_builder->add_member(m_type, m_ref, m_role); + m_next_val = next_val::none; + m_type = osmium::item_type::undefined; + m_ref = 0; + m_role.clear(); + return true; + } + + static bool null() { return true; } + static bool boolean(bool /*val*/) { return true; } + static bool binary(std::vector & /*val*/) { return true; } + static bool start_object(std::size_t /*elements*/) { return true; } + static bool start_array(std::size_t /*elements*/) { return true; } + static bool end_array() { return true; } + + static bool parse_error(std::size_t /*position*/, + const std::string & /*last_token*/, + const nlohmann::json::exception &ex) + { + throw ex; + } + +private: + osmium::builder::RelationMemberListBuilder *m_builder; + std::string m_role; + osmium::object_id_type m_ref = 0; + osmium::item_type m_type = osmium::item_type::undefined; + enum class next_val + { + none, + type, + ref, + role + } m_next_val = next_val::none; +}; + +template +void pgsql_parse_json_members(char const *string, + osmium::memory::Buffer *buffer, T *obuilder) +{ + osmium::builder::RelationMemberListBuilder builder{*buffer, obuilder}; + member_list_json_builder parser{&builder}; + nlohmann::json::sax_parse(string, &parser); +} + void pgsql_parse_members(char const *string, osmium::memory::Buffer *buffer, osmium::builder::RelationBuilder *obuilder) { @@ -220,8 +484,79 @@ } } +template +void set_attributes_on_builder(T *builder, pg_result_t const &result, int num, + int offset) +{ + if (!result.is_null(num, offset + 2)) { + builder->set_timestamp( + std::strtoul(result.get_value(num, offset + 2), nullptr, 10)); + } + if (!result.is_null(num, offset + 3)) { + builder->set_version(result.get_value(num, offset + 3)); + } + if (!result.is_null(num, offset + 4)) { + builder->set_changeset(result.get_value(num, offset + 4)); + } + if (!result.is_null(num, offset + 5)) { + builder->set_uid(result.get_value(num, offset + 5)); + } + if (!result.is_null(num, offset + 6)) { + builder->set_user(result.get_value(num, offset + 6)); + } +} + } // anonymous namespace +static void tags_to_json(osmium::TagList const &tags, json_writer_t *writer) +{ + writer->start_object(); + + for (auto const &tag : tags) { + writer->key(tag.key()); + writer->string(tag.value()); + writer->next(); + } + + writer->end_object(); +} + +static void members_to_json(osmium::RelationMemberList const &members, + json_writer_t *writer) +{ + writer->start_array(); + + for (auto const &member : members) { + writer->start_object(); + + writer->key("type"); + switch (member.type()) { + case osmium::item_type::node: + writer->string("N"); + break; + case osmium::item_type::way: + writer->string("W"); + break; + default: // osmium::item_type::relation + writer->string("R"); + break; + } + writer->next(); + + writer->key("ref"); + writer->number(member.ref()); + writer->next(); + + writer->key("role"); + writer->string(member.role()); + writer->end_object(); + + writer->next(); + } + + writer->end_array(); +} + void middle_pgsql_t::buffer_store_tags(osmium::OSMObject const &obj, bool attrs) { if (obj.tags().empty() && !attrs) { @@ -247,6 +582,45 @@ } } +void middle_pgsql_t::copy_attributes(osmium::OSMObject const &obj) +{ + if (obj.timestamp()) { + m_db_copy.add_column(obj.timestamp().to_iso()); + } else { + m_db_copy.add_null_column(); + } + + if (obj.version()) { + m_db_copy.add_column(obj.version()); + } else { + m_db_copy.add_null_column(); + } + + if (obj.changeset()) { + m_db_copy.add_columns(obj.changeset()); + } else { + m_db_copy.add_null_column(); + } + + if (obj.uid()) { + m_db_copy.add_columns(obj.uid()); + m_users.try_emplace(obj.uid(), obj.user()); + } else { + m_db_copy.add_null_column(); + } +} + +void middle_pgsql_t::copy_tags(osmium::OSMObject const &obj) +{ + if (m_store_options.db_format == 2) { + json_writer_t writer; + tags_to_json(obj.tags(), &writer); + m_db_copy.add_column(writer.json()); + return; + } + buffer_store_tags(obj, m_store_options.with_attributes); +} + std::size_t middle_query_pgsql_t::get_way_node_locations_db( osmium::WayNodeList *nodes) const { @@ -274,10 +648,10 @@ auto const res = m_sql_conn.exec_prepared("get_node_list", id_list()); std::unordered_map locs; for (int i = 0; i < res.num_tuples(); ++i) { - locs.emplace( - osmium::string_to_object_id(res.get_value(i, 0)), - osmium::Location{(int)strtol(res.get_value(i, 1), nullptr, 10), - (int)strtol(res.get_value(i, 2), nullptr, 10)}); + locs.emplace(osmium::string_to_object_id(res.get_value(i, 0)), + osmium::Location{ + (int)std::strtol(res.get_value(i, 1), nullptr, 10), + (int)std::strtol(res.get_value(i, 2), nullptr, 10)}); } for (auto &n : *nodes) { @@ -293,6 +667,8 @@ void middle_pgsql_t::node(osmium::Node const &node) { + assert(m_middle_state == middle_state::node); + if (node.deleted()) { node_delete(node.id()); } else { @@ -304,6 +680,8 @@ } void middle_pgsql_t::way(osmium::Way const &way) { + assert(m_middle_state == middle_state::way); + if (way.deleted()) { way_delete(way.id()); } else { @@ -315,6 +693,8 @@ } void middle_pgsql_t::relation(osmium::Relation const &relation) { + assert(m_middle_state == middle_state::relation); + if (relation.deleted()) { relation_delete(relation.id()); } else { @@ -329,16 +709,30 @@ { m_cache->set(node.id(), node.location()); - if (!m_options->flat_node_file.empty()) { + if (m_persistent_cache) { m_persistent_cache->set(node.id(), node.location()); - } else { - m_db_copy.new_line(m_tables.nodes().copy_target()); + } + + if (!m_store_options.nodes) { + return; + } + + if (!m_store_options.untagged_nodes && node.tags().empty()) { + return; + } - m_db_copy.add_columns(node.id(), node.location().y(), - node.location().x()); + m_db_copy.new_line(m_tables.nodes().copy_target()); - m_db_copy.finish_line(); + m_db_copy.add_columns(node.id(), node.location().y(), node.location().x()); + + if (m_store_options.db_format == 2) { + if (m_store_options.with_attributes) { + copy_attributes(node); + } + copy_tags(node); } + + m_db_copy.finish_line(); } std::size_t middle_query_pgsql_t::get_way_node_locations_flatnodes( @@ -367,8 +761,8 @@ return osmium::Location{}; } - return osmium::Location{(int)strtol(res.get_value(0, 1), nullptr, 10), - (int)strtol(res.get_value(0, 2), nullptr, 10)}; + return osmium::Location{(int)std::strtol(res.get_value(0, 1), nullptr, 10), + (int)std::strtol(res.get_value(0, 2), nullptr, 10)}; } osmium::Location @@ -401,27 +795,142 @@ { assert(m_options->append); - if (!m_options->flat_node_file.empty()) { + if (m_persistent_cache) { m_persistent_cache->set(osm_id, osmium::Location{}); - } else { + } + + if (m_store_options.nodes && osm_id <= m_tables.nodes().max_id()) { m_db_copy.new_line(m_tables.nodes().copy_target()); m_db_copy.delete_object(osm_id); } } -idlist_t middle_pgsql_t::get_ways_by_node(osmid_t osm_id) +void middle_pgsql_t::get_node_parents( + osmium::index::IdSetSmall const &changed_nodes, + osmium::index::IdSetSmall *parent_ways, + osmium::index::IdSetSmall *parent_relations) const { - return get_ids_from_db(&m_db_connection, "mark_ways_by_node", osm_id); -} + util::timer_t timer; -idlist_t middle_pgsql_t::get_rels_by_node(osmid_t osm_id) -{ - return get_ids_from_db(&m_db_connection, "mark_rels_by_node", osm_id); + m_db_connection.exec("BEGIN"); + m_db_connection.exec("CREATE TEMP TABLE osm2pgsql_changed_nodes" + " (id int8 NOT NULL) ON COMMIT DROP"); + m_db_connection.exec("CREATE TEMP TABLE osm2pgsql_changed_ways" + " (id int8 NOT NULL) ON COMMIT DROP"); + m_db_connection.exec("CREATE TEMP TABLE osm2pgsql_changed_relations" + " (id int8 NOT NULL) ON COMMIT DROP"); + + send_id_list(m_db_connection, "osm2pgsql_changed_nodes", changed_nodes); + + m_db_connection.exec("ANALYZE osm2pgsql_changed_nodes"); + + bool const has_bucket_index = + check_bucket_index(&m_db_connection, m_options->prefix); + + if (has_bucket_index) { + m_db_connection.exec(build_sql(*m_options, R"( +WITH changed_buckets AS ( + SELECT array_agg(id) AS node_ids, id >> {way_node_index_id_shift} AS bucket + FROM osm2pgsql_changed_nodes GROUP BY id >> {way_node_index_id_shift} +) +INSERT INTO osm2pgsql_changed_ways + SELECT DISTINCT w.id + FROM {schema}"{prefix}_ways" w, changed_buckets b + WHERE w.nodes && b.node_ids + AND {schema}"{prefix}_index_bucket"(w.nodes) + && ARRAY[b.bucket]; + )")); + } else { + m_db_connection.exec(build_sql(*m_options, R"( +INSERT INTO osm2pgsql_changed_ways + SELECT DISTINCT w.id + FROM {schema}"{prefix}_ways" w, osm2pgsql_changed_nodes n + WHERE w.nodes && ARRAY[n.id] + )")); + } + + if (m_options->middle_database_format == 1) { + m_db_connection.exec(build_sql(*m_options, R"( +INSERT INTO osm2pgsql_changed_relations + SELECT DISTINCT r.id + FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_nodes n + WHERE r.parts && ARRAY[n.id] + AND r.parts[1:way_off] && ARRAY[n.id] + )")); + } else { + m_db_connection.exec(build_sql(*m_options, R"( +INSERT INTO osm2pgsql_changed_relations + SELECT DISTINCT r.id + FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_nodes c + WHERE {schema}"{prefix}_member_ids"(r.members, 'N'::char) && ARRAY[c.id]; + )")); + } + + load_id_list(m_db_connection, "osm2pgsql_changed_ways", parent_ways); + load_id_list(m_db_connection, "osm2pgsql_changed_relations", + parent_relations); + + m_db_connection.exec("COMMIT"); + + timer.stop(); + + log_debug("Found {} new/changed nodes in input.", changed_nodes.size()); + log_debug(" Found in {} their {} parent ways and {} parent relations.", + std::chrono::duration_cast(timer.elapsed()), + parent_ways->size(), parent_relations->size()); } -idlist_t middle_pgsql_t::get_rels_by_way(osmid_t osm_id) +void middle_pgsql_t::get_way_parents( + osmium::index::IdSetSmall const &changed_ways, + osmium::index::IdSetSmall *parent_relations) const { - return get_ids_from_db(&m_db_connection, "mark_rels_by_way", osm_id); + util::timer_t timer; + + auto const num_relations_referenced_by_nodes = parent_relations->size(); + + m_db_connection.exec("BEGIN"); + m_db_connection.exec("CREATE TEMP TABLE osm2pgsql_changed_ways" + " (id int8 NOT NULL) ON COMMIT DROP"); + m_db_connection.exec("CREATE TEMP TABLE osm2pgsql_changed_relations" + " (id int8 NOT NULL) ON COMMIT DROP"); + + send_id_list(m_db_connection, "osm2pgsql_changed_ways", changed_ways); + + m_db_connection.exec("ANALYZE osm2pgsql_changed_ways"); + + if (m_options->middle_database_format == 1) { + m_db_connection.exec(build_sql(*m_options, R"( +INSERT INTO osm2pgsql_changed_relations + SELECT DISTINCT r.id + FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_ways w + WHERE r.parts && ARRAY[w.id] + AND r.parts[way_off+1:rel_off] && ARRAY[w.id] + )")); + } else { + m_db_connection.exec(build_sql(*m_options, R"( +INSERT INTO osm2pgsql_changed_relations + SELECT DISTINCT r.id + FROM {schema}"{prefix}_rels" r, osm2pgsql_changed_ways c + WHERE {schema}"{prefix}_member_ids"(r.members, 'W'::char) && ARRAY[c.id]; + )")); + } + + load_id_list(m_db_connection, "osm2pgsql_changed_relations", + parent_relations); + + m_db_connection.exec("COMMIT"); + + timer.stop(); + log_debug("Found {} ways that are new/changed in input or parent of" + " changed node.", + changed_ways.size()); + log_debug(" Found in {} their {} parent relations.", + std::chrono::duration_cast(timer.elapsed()), + parent_relations->size() - num_relations_referenced_by_nodes); + + // (Potentially) contains parent relations from nodes and from ways. Make + // sure they are merged. + parent_relations->sort_unique(); } void middle_pgsql_t::way_set(osmium::Way const &way) @@ -430,6 +939,10 @@ m_db_copy.add_column(way.id()); + if (m_store_options.db_format == 2 && m_store_options.with_attributes) { + copy_attributes(way); + } + // nodes m_db_copy.new_array(); for (auto const &n : way.nodes()) { @@ -437,11 +950,38 @@ } m_db_copy.finish_array(); - buffer_store_tags(way, m_options->extra_attributes); + copy_tags(way); m_db_copy.finish_line(); } +/** + * Build way in buffer from database results. + */ +static void build_way(osmid_t id, pg_result_t const &res, int res_num, + int offset, osmium::memory::Buffer *buffer, + uint8_t db_format, bool with_attributes) +{ + osmium::builder::WayBuilder builder{*buffer}; + builder.set_id(id); + + if (db_format == 1) { + if (with_attributes) { + pgsql_get_attr_from_tags(res.get_value(res_num, offset + 1), + &builder); + } + pgsql_parse_nodes(res.get_value(res_num, offset + 0), buffer, &builder); + pgsql_parse_tags(res.get_value(res_num, offset + 1), buffer, &builder); + return; + } + + if (with_attributes) { + set_attributes_on_builder(&builder, res, res_num, offset); + } + pgsql_parse_nodes(res.get_value(res_num, offset + 0), buffer, &builder); + pgsql_parse_json_tags(res.get_value(res_num, offset + 1), buffer, &builder); +} + bool middle_query_pgsql_t::way_get(osmid_t id, osmium::memory::Buffer *buffer) const { @@ -453,13 +993,8 @@ return false; } - { - osmium::builder::WayBuilder builder{*buffer}; - builder.set_id(id); - - pgsql_parse_nodes(res.get_value(0, 0), buffer, &builder); - pgsql_parse_tags(res.get_value(0, 1), buffer, &builder); - } + build_way(id, res, 0, 0, buffer, m_store_options.db_format, + m_store_options.with_attributes); buffer->commit(); @@ -505,12 +1040,9 @@ // back to the list of ways given by the caller for (int j = 0; j < res.num_tuples(); ++j) { if (member.ref() == wayidspg[static_cast(j)]) { - osmium::builder::WayBuilder builder{*buffer}; - builder.set_id(member.ref()); - - pgsql_parse_nodes(res.get_value(j, 1), buffer, &builder); - pgsql_parse_tags(res.get_value(j, 2), buffer, &builder); - + build_way(member.ref(), res, j, 1, buffer, + m_store_options.db_format, + m_store_options.with_attributes); ++members_found; break; } @@ -526,17 +1058,20 @@ void middle_pgsql_t::way_delete(osmid_t osm_id) { assert(m_options->append); - m_db_copy.new_line(m_tables.ways().copy_target()); - m_db_copy.delete_object(osm_id); + + if (osm_id <= m_tables.ways().max_id()) { + m_db_copy.new_line(m_tables.ways().copy_target()); + m_db_copy.delete_object(osm_id); + } } -void middle_pgsql_t::relation_set(osmium::Relation const &rel) +void middle_pgsql_t::relation_set_format1(osmium::Relation const &rel) { // Sort relation members by their type. - idlist_t parts[3]; + std::array parts; for (auto const &m : rel.members()) { - parts[osmium::item_type_to_nwr_index(m.type())].push_back(m.ref()); + parts.at(osmium::item_type_to_nwr_index(m.type())).push_back(m.ref()); } m_db_copy.new_line(m_tables.relations().copy_target()); @@ -573,14 +1108,38 @@ m_db_copy.finish_line(); } -bool middle_query_pgsql_t::relation_get(osmid_t id, - osmium::memory::Buffer *buffer) const +void middle_pgsql_t::relation_set_format2(osmium::Relation const &rel) +{ + m_db_copy.new_line(m_tables.relations().copy_target()); + m_db_copy.add_column(rel.id()); + + if (m_store_options.with_attributes) { + copy_attributes(rel); + } + + json_writer_t writer; + members_to_json(rel.members(), &writer); + m_db_copy.add_column(writer.json()); + + copy_tags(rel); + + m_db_copy.finish_line(); +} + +void middle_pgsql_t::relation_set(osmium::Relation const &rel) +{ + if (m_store_options.db_format == 2) { + return relation_set_format2(rel); + } + return relation_set_format1(rel); +} + +bool middle_query_pgsql_t::relation_get_format1( + osmid_t id, osmium::memory::Buffer *buffer) const { assert(buffer); auto const res = m_sql_conn.exec_prepared("get_rel", id); - // Fields are: members, tags, member_count */ - // if (res.num_tuples() != 1) { return false; } @@ -588,7 +1147,11 @@ { osmium::builder::RelationBuilder builder{*buffer}; builder.set_id(id); + if (m_store_options.with_attributes) { + pgsql_get_attr_from_tags(res.get_value(0, 1), &builder); + } + pgsql_get_attr_from_tags(res.get_value(0, 1), &builder); pgsql_parse_members(res.get_value(0, 0), buffer, &builder); pgsql_parse_tags(res.get_value(0, 1), buffer, &builder); } @@ -598,18 +1161,61 @@ return true; } +bool middle_query_pgsql_t::relation_get_format2( + osmid_t id, osmium::memory::Buffer *buffer) const +{ + assert(buffer); + + auto const res = m_sql_conn.exec_prepared("get_rel", id); + + if (res.num_tuples() == 0) { + return false; + } + + { + osmium::builder::RelationBuilder builder{*buffer}; + builder.set_id(id); + + if (m_store_options.with_attributes) { + set_attributes_on_builder(&builder, res, 0, 0); + } + + pgsql_parse_json_members(res.get_value(0, 0), buffer, &builder); + pgsql_parse_json_tags(res.get_value(0, 1), buffer, &builder); + } + buffer->commit(); + + return true; +} + +bool middle_query_pgsql_t::relation_get(osmid_t id, + osmium::memory::Buffer *buffer) const +{ + if (m_store_options.db_format == 2) { + return relation_get_format2(id, buffer); + } + return relation_get_format1(id, buffer); +} + void middle_pgsql_t::relation_delete(osmid_t osm_id) { assert(m_options->append); - m_db_copy.new_line(m_tables.relations().copy_target()); - m_db_copy.delete_object(osm_id); + if (osm_id <= m_tables.relations().max_id()) { + m_db_copy.new_line(m_tables.relations().copy_target()); + m_db_copy.delete_object(osm_id); + } } void middle_pgsql_t::after_nodes() { + assert(m_middle_state == middle_state::node); +#ifndef NDEBUG + m_middle_state = middle_state::way; +#endif + m_db_copy.sync(); - if (!m_options->append && m_options->flat_node_file.empty()) { + if (!m_options->append && m_store_options.nodes) { auto const &table = m_tables.nodes(); analyze_table(m_db_connection, table.schema(), table.name()); } @@ -617,6 +1223,11 @@ void middle_pgsql_t::after_ways() { + assert(m_middle_state == middle_state::way); +#ifndef NDEBUG + m_middle_state = middle_state::relation; +#endif + m_db_copy.sync(); if (!m_options->append) { auto const &table = m_tables.ways(); @@ -626,21 +1237,36 @@ void middle_pgsql_t::after_relations() { + assert(m_middle_state == middle_state::relation); +#ifndef NDEBUG + m_middle_state = middle_state::done; +#endif + m_db_copy.sync(); if (!m_options->append) { auto const &table = m_tables.relations(); analyze_table(m_db_connection, table.schema(), table.name()); } + if (m_store_options.db_format == 2 && m_store_options.with_attributes && + !m_options->droptemp) { + if (m_append) { + update_users_table(); + } else { + write_users_table(); + } + } + // release the copy thread and its database connection m_copy_thread->finish(); } middle_query_pgsql_t::middle_query_pgsql_t( std::string const &conninfo, std::shared_ptr cache, - std::shared_ptr persistent_cache) + std::shared_ptr persistent_cache, + middle_pgsql_options const &options) : m_sql_conn(conninfo), m_cache(std::move(cache)), - m_persistent_cache(std::move(persistent_cache)) + m_persistent_cache(std::move(persistent_cache)), m_store_options(options) { // Disable JIT and parallel workers as they are known to cause // problems when accessing the intarrays. @@ -648,39 +1274,95 @@ m_sql_conn.set_config("max_parallel_workers_per_gather", "0"); } +static void table_setup(pg_conn_t const &db_connection, + middle_pgsql_t::table_desc const &table) +{ + log_debug("Setting up table '{}'", table.name()); + auto const qual_name = qualified_name(table.schema(), table.name()); + db_connection.exec("DROP TABLE IF EXISTS {} CASCADE", qual_name); + if (!table.m_create_table.empty()) { + db_connection.exec(table.m_create_table); + } +} + void middle_pgsql_t::start() { + assert(m_middle_state == middle_state::constructed); +#ifndef NDEBUG + m_middle_state = middle_state::node; +#endif + if (m_options->append) { // Disable JIT and parallel workers as they are known to cause // problems when accessing the intarrays. m_db_connection.set_config("jit_above_cost", "-1"); m_db_connection.set_config("max_parallel_workers_per_gather", "0"); - // Prepare queries for updating dependent objects - for (auto &table : m_tables) { - if (!table.m_prepare_fw_dep_lookups.empty()) { - m_db_connection.exec(table.m_prepare_fw_dep_lookups); - } + // Remember the maximum OSM ids in the middle tables. This is a very + // fast operation due to the index on the table. Later when we need + // to delete entries, we don't have to bother with entries that are + // definitely not in the table. + if (m_store_options.nodes) { + m_tables.nodes().init_max_id(m_db_connection); } + m_tables.ways().init_max_id(m_db_connection); + m_tables.relations().init_max_id(m_db_connection); } else { - m_db_connection.exec("SET client_min_messages = WARNING"); + if (m_store_options.db_format == 2) { + table_setup(m_db_connection, m_users_table); + } for (auto const &table : m_tables) { - log_debug("Setting up table '{}'", table.name()); - auto const qual_name = qualified_name(table.schema(), table.name()); - m_db_connection.exec("DROP TABLE IF EXISTS {} CASCADE", qual_name); - if (!table.m_create_table.empty()) { - m_db_connection.exec(table.m_create_table); - } + table_setup(m_db_connection, table); } } } +void middle_pgsql_t::write_users_table() +{ + log_info("Writing {} entries to table '{}'...", m_users.size(), + m_users_table.name()); + auto const users_table = std::make_shared( + m_users_table.schema(), m_users_table.name(), "id"); + + for (auto const &[id, name] : m_users) { + m_db_copy.new_line(users_table); + m_db_copy.add_columns(id, name); + m_db_copy.finish_line(); + } + m_db_copy.sync(); + + m_users.clear(); + + analyze_table(m_db_connection, m_users_table.schema(), + m_users_table.name()); +} + +void middle_pgsql_t::update_users_table() +{ + log_info("Writing {} entries to table '{}'...", m_users.size(), + m_users_table.name()); + + m_db_connection.exec("PREPARE insert_user(int8, text) AS" + " INSERT INTO {}\"{}\" (id, name) VALUES ($1, $2)" + " ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id", + m_users_table.schema(), m_users_table.name()); + + for (auto const &[id, name] : m_users) { + m_db_connection.exec_prepared("insert_user", id, name); + } + + m_users.clear(); + + analyze_table(m_db_connection, m_users_table.schema(), + m_users_table.name()); +} + void middle_pgsql_t::stop() { + assert(m_middle_state == middle_state::done); + m_cache.reset(); - if (!m_options->flat_node_file.empty()) { - m_persistent_cache.reset(); - } + m_persistent_cache.reset(); if (m_options->droptemp) { // Dropping the tables is fast, so do it synchronously to guarantee @@ -706,7 +1388,23 @@ } } -static table_sql sql_for_nodes(bool create_table) noexcept +static table_sql sql_for_users(middle_pgsql_options const &store_options) +{ + table_sql sql{}; + + sql.name = "{prefix}_users"; + + if (store_options.with_attributes) { + sql.create_table = "CREATE TABLE {schema}\"{prefix}_users\" (" + " id INT4 PRIMARY KEY {using_tablespace}," + " name TEXT NOT NULL" + ") {data_tablespace}"; + } + + return sql; +} + +static table_sql sql_for_nodes_format1(bool create_table) { table_sql sql{}; @@ -718,22 +1416,49 @@ " id int8 PRIMARY KEY {using_tablespace}," " lat int4 NOT NULL," " lon int4 NOT NULL" - ") {data_tablespace};\n"; + ") {data_tablespace}"; - sql.prepare_query = + sql.prepare_queries = { "PREPARE get_node_list(int8[]) AS" " SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\"" - " WHERE id = ANY($1::int8[]);\n" + " WHERE id = ANY($1::int8[])", "PREPARE get_node(int8) AS" " SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\"" - " WHERE id = $1;\n"; + " WHERE id = $1"}; + } + + return sql; +} + +static table_sql sql_for_nodes_format2(middle_pgsql_options const &options) +{ + table_sql sql{}; + + sql.name = "{prefix}_nodes"; + + if (options.nodes) { + sql.create_table = + "CREATE {unlogged} TABLE {schema}\"{prefix}_nodes\" (" + " id int8 PRIMARY KEY {using_tablespace}," + " lat int4 NOT NULL," + " lon int4 NOT NULL," + "{attribute_columns_definition}" + " tags jsonb NOT NULL" + ") {data_tablespace}"; + + sql.prepare_queries = { + "PREPARE get_node_list(int8[]) AS" + " SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\"" + " WHERE id = ANY($1::int8[])", + "PREPARE get_node(int8) AS" + " SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\"" + " WHERE id = $1"}; } return sql; } -static table_sql sql_for_ways(bool has_bucket_index, - uint8_t way_node_index_id_shift) noexcept +static table_sql sql_for_ways_format1(uint8_t way_node_index_id_shift) { table_sql sql{}; @@ -743,52 +1468,83 @@ " id int8 PRIMARY KEY {using_tablespace}," " nodes int8[] NOT NULL," " tags text[]" - ") {data_tablespace};\n"; + ") {data_tablespace}"; - sql.prepare_query = "PREPARE get_way(int8) AS" - " SELECT nodes, tags" - " FROM {schema}\"{prefix}_ways\" WHERE id = $1;\n" - "PREPARE get_way_list(int8[]) AS" - " SELECT id, nodes, tags" - " FROM {schema}\"{prefix}_ways\"" - " WHERE id = ANY($1::int8[]);\n"; + sql.prepare_queries = {"PREPARE get_way(int8) AS" + " SELECT nodes, tags" + " FROM {schema}\"{prefix}_ways\" WHERE id = $1", + "PREPARE get_way_list(int8[]) AS" + " SELECT id, nodes, tags" + " FROM {schema}\"{prefix}_ways\"" + " WHERE id = ANY($1::int8[])"}; - if (has_bucket_index) { - sql.prepare_fw_dep_lookups = - "PREPARE mark_ways_by_node(int8) AS" - " SELECT id FROM {schema}\"{prefix}_ways\" w" - " WHERE $1 = ANY(nodes)" - " AND {schema}\"{prefix}_index_bucket\"(w.nodes)" - " && {schema}\"{prefix}_index_bucket\"(ARRAY[$1]);\n"; - } else { - sql.prepare_fw_dep_lookups = - "PREPARE mark_ways_by_node(int8) AS" - " SELECT id FROM {schema}\"{prefix}_ways\"" - " WHERE nodes && ARRAY[$1];\n"; + if (way_node_index_id_shift == 0) { + sql.create_fw_dep_indexes = { + "CREATE INDEX ON {schema}\"{prefix}_ways\" USING GIN (nodes)" + " WITH (fastupdate = off) {index_tablespace}"}; + } else { + sql.create_fw_dep_indexes = { + "CREATE OR REPLACE FUNCTION" + " {schema}\"{prefix}_index_bucket\"(int8[])" + " RETURNS int8[] AS $$" + " SELECT ARRAY(SELECT DISTINCT" + " unnest($1) >> {way_node_index_id_shift})" + "$$ LANGUAGE SQL IMMUTABLE", + "CREATE INDEX \"{prefix}_ways_nodes_bucket_idx\"" + " ON {schema}\"{prefix}_ways\"" + " USING GIN ({schema}\"{prefix}_index_bucket\"(nodes))" + " WITH (fastupdate = off) {index_tablespace}"}; } - if (way_node_index_id_shift == 0) { - sql.create_fw_dep_indexes = + return sql; +} + +static table_sql sql_for_ways_format2(middle_pgsql_options const &options) +{ + table_sql sql{}; + + sql.name = "{prefix}_ways"; + + sql.create_table = "CREATE {unlogged} TABLE {schema}\"{prefix}_ways\" (" + " id int8 PRIMARY KEY {using_tablespace}," + "{attribute_columns_definition}" + " nodes int8[] NOT NULL," + " tags jsonb NOT NULL" + ") {data_tablespace}"; + + sql.prepare_queries = {"PREPARE get_way(int8) AS" + " SELECT nodes, tags{attribute_columns_use}" + " FROM {schema}\"{prefix}_ways\" o" + " {users_table_access}" + " WHERE o.id = $1", + "PREPARE get_way_list(int8[]) AS" + " SELECT o.id, nodes, tags{attribute_columns_use}" + " FROM {schema}\"{prefix}_ways\" o" + " {users_table_access}" + " WHERE o.id = ANY($1::int8[])"}; + + if (options.way_node_index_id_shift == 0) { + sql.create_fw_dep_indexes = { "CREATE INDEX ON {schema}\"{prefix}_ways\" USING GIN (nodes)" - " WITH (fastupdate = off) {index_tablespace};\n"; + " WITH (fastupdate = off) {index_tablespace}"}; } else { - sql.create_fw_dep_indexes = + sql.create_fw_dep_indexes = { "CREATE OR REPLACE FUNCTION" " {schema}\"{prefix}_index_bucket\"(int8[])" - " RETURNS int8[] AS $$\n" + " RETURNS int8[] AS $$" " SELECT ARRAY(SELECT DISTINCT" - " unnest($1) >> {way_node_index_id_shift})\n" - "$$ LANGUAGE SQL IMMUTABLE;\n" + " unnest($1) >> {way_node_index_id_shift})" + "$$ LANGUAGE SQL IMMUTABLE", "CREATE INDEX \"{prefix}_ways_nodes_bucket_idx\"" " ON {schema}\"{prefix}_ways\"" " USING GIN ({schema}\"{prefix}_index_bucket\"(nodes))" - " WITH (fastupdate = off) {index_tablespace};\n"; + " WITH (fastupdate = off) {index_tablespace}"}; } return sql; } -static table_sql sql_for_relations() noexcept +static table_sql sql_for_relations_format1() { table_sql sql{}; @@ -801,38 +1557,56 @@ " parts int8[]," " members text[]," " tags text[]" - ") {data_tablespace};\n"; + ") {data_tablespace}"; - sql.prepare_query = "PREPARE get_rel(int8) AS" - " SELECT members, tags" - " FROM {schema}\"{prefix}_rels\" WHERE id = $1;\n"; - - sql.prepare_fw_dep_lookups = - "PREPARE mark_rels_by_node(int8) AS" - " SELECT id FROM {schema}\"{prefix}_rels\"" - " WHERE parts && ARRAY[$1]" - " AND parts[1:way_off] && ARRAY[$1];\n" - "PREPARE mark_rels_by_way(int8) AS" - " SELECT id FROM {schema}\"{prefix}_rels\"" - " WHERE parts && ARRAY[$1]" - " AND parts[way_off+1:rel_off] && ARRAY[$1];\n"; + sql.prepare_queries = {"PREPARE get_rel(int8) AS" + " SELECT members, tags" + " FROM {schema}\"{prefix}_rels\" WHERE id = $1"}; - sql.create_fw_dep_indexes = + sql.create_fw_dep_indexes = { "CREATE INDEX ON {schema}\"{prefix}_rels\" USING GIN (parts)" - " WITH (fastupdate = off) {index_tablespace};\n"; + " WITH (fastupdate = off) {index_tablespace}"}; return sql; } -static bool check_bucket_index(pg_conn_t *db_connection, - std::string const &prefix) +static table_sql sql_for_relations_format2() { - auto const res = - db_connection->exec("SELECT relname FROM pg_class" - " WHERE relkind='i'" - " AND relname = '{}_ways_nodes_bucket_idx'", - prefix); - return res.num_tuples() > 0; + table_sql sql{}; + + sql.name = "{prefix}_rels"; + + sql.create_table = "CREATE {unlogged} TABLE {schema}\"{prefix}_rels\" (" + " id int8 PRIMARY KEY {using_tablespace}," + "{attribute_columns_definition}" + " members jsonb NOT NULL," + " tags jsonb NOT NULL" + ") {data_tablespace}"; + + sql.prepare_queries = {"PREPARE get_rel(int8) AS" + " SELECT members, tags{attribute_columns_use}" + " FROM {schema}\"{prefix}_rels\" o" + " {users_table_access}" + " WHERE o.id = $1"}; + + sql.create_fw_dep_indexes = { + "CREATE OR REPLACE FUNCTION" + " {schema}\"{prefix}_member_ids\"(jsonb, char)" + " RETURNS int8[] AS $$" + " SELECT array_agg((el->>'ref')::int8)" + " FROM jsonb_array_elements($1) AS el" + " WHERE el->>'type' = $2" + "$$ LANGUAGE SQL IMMUTABLE", + "CREATE INDEX \"{prefix}_rels_node_members_idx\"" + " ON {schema}\"{prefix}_rels\" USING GIN" + " (({schema}\"{prefix}_member_ids\"(members, 'N'::char)))" + " WITH (fastupdate = off) {index_tablespace}", + "CREATE INDEX \"{prefix}_rels_way_members_idx\"" + " ON {schema}\"{prefix}_rels\" USING GIN" + " (({schema}\"{prefix}_member_ids\"(members, 'W'::char)))" + " WITH (fastupdate = off) {index_tablespace}"}; + + return sql; } middle_pgsql_t::middle_pgsql_t(std::shared_ptr thread_pool, @@ -842,14 +1616,27 @@ static_cast(options->cache) * 1024UL * 1024UL)), m_db_connection(m_options->conninfo), m_copy_thread(std::make_shared(options->conninfo)), - m_db_copy(m_copy_thread) + m_db_copy(m_copy_thread), m_append(options->append) { - if (!options->flat_node_file.empty()) { + m_store_options.with_attributes = options->extra_attributes; + m_store_options.db_format = options->middle_database_format; + m_store_options.way_node_index_id_shift = options->way_node_index_id_shift; + + if (m_store_options.db_format == 2 && options->middle_with_nodes) { + m_store_options.nodes = true; + } + + if (options->flat_node_file.empty()) { + m_store_options.nodes = true; + m_store_options.untagged_nodes = true; + } else { + m_store_options.use_flat_node_file = true; m_persistent_cache = std::make_shared( options->flat_node_file, options->droptemp); } - log_debug("Mid: pgsql, cache={}", options->cache); + log_debug("Mid: pgsql, cache={}, db_format={}", options->cache, + options->middle_database_format); bool const has_bucket_index = check_bucket_index(&m_db_connection, options->prefix); @@ -859,26 +1646,56 @@ log_debug("You don't have a bucket index. See manual for details."); } - m_tables.nodes() = - table_desc{*options, sql_for_nodes(options->flat_node_file.empty())}; - m_tables.ways() = - table_desc{*options, sql_for_ways(has_bucket_index, - options->way_node_index_id_shift)}; - m_tables.relations() = table_desc{*options, sql_for_relations()}; + if (m_store_options.db_format == 1) { + m_tables.nodes() = table_desc{ + *options, + sql_for_nodes_format1(!m_store_options.use_flat_node_file)}; + + m_tables.ways() = table_desc{ + *options, + sql_for_ways_format1(m_store_options.way_node_index_id_shift)}; + + m_tables.relations() = + table_desc{*options, sql_for_relations_format1()}; + } else { + m_tables.nodes() = + table_desc{*options, sql_for_nodes_format2(m_store_options)}; + + m_tables.ways() = + table_desc{*options, sql_for_ways_format2(m_store_options)}; + + m_tables.relations() = + table_desc{*options, sql_for_relations_format2()}; + + m_users_table = table_desc{*options, sql_for_users(m_store_options)}; + } +} + +void middle_pgsql_t::set_requirements( + output_requirements const & /*requirements*/) +{ + log_debug("Middle 'pgsql' options:"); + log_debug(" nodes: {}", m_store_options.nodes); + log_debug(" untagged_nodes: {}", m_store_options.untagged_nodes); + log_debug(" db_format: {}", m_store_options.db_format); + log_debug(" use_flat_node_file: {}", m_store_options.use_flat_node_file); + log_debug(" way_node_index_id_shift: {}", + m_store_options.way_node_index_id_shift); + log_debug(" with_attributes: {}", m_store_options.with_attributes); } std::shared_ptr middle_pgsql_t::get_query_instance() { - // NOTE: this is thread safe for use in pending async processing only because - // during that process they are only read from + // NOTE: this is thread safe for use in pending async processing only + // because during that process they are only read from auto mid = std::make_unique( - m_options->conninfo, m_cache, m_persistent_cache); + m_options->conninfo, m_cache, m_persistent_cache, m_store_options); // We use a connection per table to enable the use of COPY for (auto &table : m_tables) { - if (!table.m_prepare_query.empty()) { - mid->exec_sql(table.m_prepare_query); + for (auto const &query : table.m_prepare_queries) { + mid->exec_sql(query); } } diff -Nru osm2pgsql-1.8.1+ds/src/middle-pgsql.hpp osm2pgsql-1.9.0+ds/src/middle-pgsql.hpp --- osm2pgsql-1.8.1+ds/src/middle-pgsql.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/middle-pgsql.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -18,6 +18,7 @@ * emit the final geometry-enabled output formats */ +#include #include #include @@ -28,14 +29,35 @@ class node_locations_t; class node_persistent_cache; -class options_t; + +struct middle_pgsql_options +{ + // Store nodes in database. + bool nodes = false; + + // Store untagged nodes also (set in addition to nodes=true). + bool untagged_nodes = false; + + // Bit shift used in way node index + uint8_t way_node_index_id_shift = 5; + + // Database format (legacy = 1, new = 2) + uint8_t db_format = 1; + + // Use a flat node file + bool use_flat_node_file = false; + + // Store attributes (timestamp, version, changeset id, user id, user name) + bool with_attributes = false; +}; class middle_query_pgsql_t : public middle_query_t { public: middle_query_pgsql_t( std::string const &conninfo, std::shared_ptr cache, - std::shared_ptr persistent_cache); + std::shared_ptr persistent_cache, + middle_pgsql_options const &options); osmium::Location get_node_location(osmid_t id) const override; @@ -47,6 +69,9 @@ osmium::memory::Buffer *buffer, osmium::osm_entity_bits::type types) const override; + bool relation_get_format1(osmid_t id, osmium::memory::Buffer *buffer) const; + bool relation_get_format2(osmid_t id, osmium::memory::Buffer *buffer) const; + bool relation_get(osmid_t id, osmium::memory::Buffer *buffer) const override; @@ -61,14 +86,16 @@ pg_conn_t m_sql_conn; std::shared_ptr m_cache; std::shared_ptr m_persistent_cache; + + middle_pgsql_options m_store_options; }; -struct table_sql { - char const *name = ""; - char const *create_table = ""; - char const *prepare_query = ""; - char const *prepare_fw_dep_lookups = ""; - char const *create_fw_dep_indexes = ""; +struct table_sql +{ + std::string name; + std::string create_table; + std::vector prepare_queries; + std::vector create_fw_dep_indexes; }; struct middle_pgsql_t : public middle_t @@ -89,9 +116,14 @@ void after_ways() override; void after_relations() override; - idlist_t get_ways_by_node(osmid_t osm_id) override; - idlist_t get_rels_by_node(osmid_t osm_id) override; - idlist_t get_rels_by_way(osmid_t osm_id) override; + void get_node_parents( + osmium::index::IdSetSmall const &changed_nodes, + osmium::index::IdSetSmall *parent_ways, + osmium::index::IdSetSmall *parent_relations) const override; + + void get_way_parents( + osmium::index::IdSetSmall const &changed_ways, + osmium::index::IdSetSmall *parent_relations) const override; class table_desc { @@ -101,10 +133,13 @@ std::string const &schema() const noexcept { - return m_copy_target->schema; + return m_copy_target->schema(); } - std::string const &name() const noexcept { return m_copy_target->name; } + std::string const &name() const noexcept + { + return m_copy_target->name(); + } std::shared_ptr const ©_target() const noexcept { @@ -118,9 +153,8 @@ void build_index(std::string const &conninfo) const; std::string m_create_table; - std::string m_prepare_query; - std::string m_prepare_fw_dep_lookups; - std::string m_create_fw_dep_indexes; + std::vector m_prepare_queries; + std::vector m_create_fw_dep_indexes; void task_set(std::future &&future) { @@ -129,13 +163,22 @@ std::chrono::microseconds task_wait() { return m_task_result.wait(); } + void init_max_id(pg_conn_t const &db_connection); + + osmid_t max_id() const noexcept { return m_max_id; } + private: std::shared_ptr m_copy_target; task_result_t m_task_result; + + /// The maximum id in the table (used only in append mode) + osmid_t m_max_id = 0; }; std::shared_ptr get_query_instance() override; + void set_requirements(output_requirements const &requirements) override; + private: void node_set(osmium::Node const &node); void node_delete(osmid_t id); @@ -143,12 +186,23 @@ void way_set(osmium::Way const &way); void way_delete(osmid_t id); + void relation_set_format1(osmium::Relation const &rel); + void relation_set_format2(osmium::Relation const &rel); + void relation_set(osmium::Relation const &rel); void relation_delete(osmid_t id); void buffer_store_tags(osmium::OSMObject const &obj, bool attrs); + void copy_attributes(osmium::OSMObject const &obj); + void copy_tags(osmium::OSMObject const &obj); + + void write_users_table(); + void update_users_table(); + + std::map m_users; osmium::nwr_array m_tables; + table_desc m_users_table; options_t const *m_options; @@ -160,6 +214,11 @@ // middle keeps its own thread for writing to the database. std::shared_ptr m_copy_thread; db_copy_mgr_t m_db_copy; + + /// Options for this middle. + middle_pgsql_options m_store_options; + + bool m_append; }; #endif // OSM2PGSQL_MIDDLE_PGSQL_HPP diff -Nru osm2pgsql-1.8.1+ds/src/middle-ram.cpp osm2pgsql-1.9.0+ds/src/middle-ram.cpp --- osm2pgsql-1.8.1+ds/src/middle-ram.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/middle-ram.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -61,6 +61,8 @@ void middle_ram_t::stop() { + assert(m_middle_state == middle_state::done); + auto const mbyte = 1024 * 1024; log_debug("Middle 'ram': Node locations: size={} bytes={}M", @@ -148,6 +150,7 @@ void middle_ram_t::node(osmium::Node const &node) { + assert(m_middle_state == middle_state::node); assert(node.visible()); if (m_store_options.locations) { @@ -162,6 +165,7 @@ void middle_ram_t::way(osmium::Way const &way) { + assert(m_middle_state == middle_state::way); assert(way.visible()); if (m_store_options.way_nodes) { @@ -177,6 +181,7 @@ void middle_ram_t::relation(osmium::Relation const &relation) { + assert(m_middle_state == middle_state::relation); assert(relation.visible()); if (m_store_options.relations) { diff -Nru osm2pgsql-1.8.1+ds/src/middle-ram.hpp osm2pgsql-1.9.0+ds/src/middle-ram.hpp --- osm2pgsql-1.8.1+ds/src/middle-ram.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/middle-ram.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -24,7 +24,6 @@ #include #include -class options_t; class thread_pool_t; /** @@ -47,7 +46,14 @@ ~middle_ram_t() noexcept override = default; - void start() override {} + void start() override + { + assert(m_middle_state == middle_state::constructed); +#ifndef NDEBUG + m_middle_state = middle_state::node; +#endif + } + void stop() override; void node(osmium::Node const &node) override; @@ -89,7 +95,7 @@ // Store ways (with tags, attributes, and way nodes) in object store. bool ways = false; - // Store relations (with tags, attribtes, and members) in object store. + // Store relations (with tags, attributes, and members) in object store. bool relations = false; }; @@ -109,7 +115,7 @@ /// Buffer for all OSM objects we store. osmium::memory::Buffer m_object_buffer{ - 1024 * 1024, osmium::memory::Buffer::auto_grow::yes}; + 1024UL * 1024UL, osmium::memory::Buffer::auto_grow::yes}; /// Indexes into object buffer. osmium::nwr_array m_object_index; diff -Nru osm2pgsql-1.8.1+ds/src/node-locations.hpp osm2pgsql-1.9.0+ds/src/node-locations.hpp --- osm2pgsql-1.8.1+ds/src/node-locations.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/node-locations.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -85,7 +85,7 @@ /// The maximum number of bytes an entry will need in storage. constexpr static std::size_t max_bytes_per_entry() noexcept { - return 10U /*max varint length*/ * 3U /*id, x, y*/; + return 10UL /*max varint length*/ * 3UL /*id, x, y*/; } bool will_resize() const noexcept diff -Nru osm2pgsql-1.8.1+ds/src/options.cpp osm2pgsql-1.9.0+ds/src/options.cpp --- osm2pgsql-1.8.1+ds/src/options.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/options.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,770 +0,0 @@ -/** - * SPDX-License-Identifier: GPL-2.0-or-later - * - * This file is part of osm2pgsql (https://osm2pgsql.org/). - * - * Copyright (C) 2006-2023 by the osm2pgsql developer community. - * For a full list of authors see the git log. - */ - -#include "format.hpp" -#include "logging.hpp" -#include "options.hpp" -#include "pgsql.hpp" -#include "reprojection.hpp" -#include "util.hpp" -#include "version.hpp" - -#include -#include -#include -#include -#include -#include -#include // for number of threads - -#ifdef HAVE_LUA -extern "C" -{ -#include -} -#endif - -#ifdef HAVE_LUAJIT -extern "C" -{ -#include -} -#endif - -static char const *program_name(char const *name) -{ - char const *const slash = std::strrchr(name, '/'); - return slash ? (slash + 1) : name; -} - -namespace { -char const *const short_options = - "ab:cd:KhlmMp:suvU:WH:P:i:IE:C:S:e:o:O:xkjGz:r:VF:"; - -struct option const long_options[] = { - {"append", no_argument, nullptr, 'a'}, - {"bbox", required_argument, nullptr, 'b'}, - {"cache", required_argument, nullptr, 'C'}, - {"cache-strategy", required_argument, nullptr, 204}, - {"create", no_argument, nullptr, 'c'}, - {"database", required_argument, nullptr, 'd'}, - {"disable-parallel-indexing", no_argument, nullptr, 'I'}, - {"drop", no_argument, nullptr, 206}, - {"expire-bbox-size", required_argument, nullptr, 214}, - {"expire-output", required_argument, nullptr, 'o'}, - {"expire-tiles", required_argument, nullptr, 'e'}, - {"extra-attributes", no_argument, nullptr, 'x'}, - {"flat-nodes", required_argument, nullptr, 'F'}, - {"help", no_argument, nullptr, 'h'}, - {"host", required_argument, nullptr, 'H'}, - {"hstore", no_argument, nullptr, 'k'}, - {"hstore-add-index", no_argument, nullptr, 211}, - {"hstore-all", no_argument, nullptr, 'j'}, - {"hstore-column", required_argument, nullptr, 'z'}, - {"hstore-match-only", no_argument, nullptr, 208}, - {"input-reader", required_argument, nullptr, 'r'}, - {"keep-coastlines", no_argument, nullptr, 'K'}, - {"latlong", no_argument, nullptr, 'l'}, - {"log-level", required_argument, nullptr, 400}, - {"log-progress", required_argument, nullptr, 401}, - {"log-sql", no_argument, nullptr, 402}, - {"log-sql-data", no_argument, nullptr, 403}, - {"merc", no_argument, nullptr, 'm'}, - {"middle-schema", required_argument, nullptr, 215}, - {"middle-way-node-index-id-shift", required_argument, nullptr, 300}, - {"multi-geometry", no_argument, nullptr, 'G'}, - {"number-processes", required_argument, nullptr, 205}, - {"output", required_argument, nullptr, 'O'}, - {"output-pgsql-schema", required_argument, nullptr, 216}, - {"password", no_argument, nullptr, 'W'}, - {"port", required_argument, nullptr, 'P'}, - {"prefix", required_argument, nullptr, 'p'}, - {"proj", required_argument, nullptr, 'E'}, - {"reproject-area", no_argument, nullptr, 213}, - {"slim", no_argument, nullptr, 's'}, - {"style", required_argument, nullptr, 'S'}, - {"tablespace-index", required_argument, nullptr, 'i'}, - {"tablespace-main-data", required_argument, nullptr, 202}, - {"tablespace-main-index", required_argument, nullptr, 203}, - {"tablespace-slim-data", required_argument, nullptr, 200}, - {"tablespace-slim-index", required_argument, nullptr, 201}, - {"tag-transform-script", required_argument, nullptr, 212}, - {"username", required_argument, nullptr, 'U'}, - {"verbose", no_argument, nullptr, 'v'}, - {"version", no_argument, nullptr, 'V'}, - {"with-forward-dependencies", required_argument, nullptr, 217}, - {nullptr, 0, nullptr, 0}}; - -void long_usage(char const *arg0, bool verbose) -{ - char const *const name = program_name(arg0); - - fmt::print(stdout, "\nUsage: {} [OPTIONS] OSM-FILE...\n", name); - (void)std::fputs( - "\nImport data from the OSM file(s) into a PostgreSQL database.\n\n\ -Full documentation is available at https://osm2pgsql.org/\n\n", - stdout); - - (void)std::fputs("\ -Common options:\n\ - -a|--append Update existing osm2pgsql database with data from file.\n\ - -c|--create Import OSM data from file into database. This is the\n\ - default if --append is not specified.\n\ - -O|--output=OUTPUT Set output. Options are:\n\ - pgsql - Output to a PostGIS database (default)\n\ - flex - More flexible output to PostGIS database\n\ - gazetteer - Output to a PostGIS database for Nominatim\n\ - (deprecated)\n\ - null - No output. Used for testing.\n\ - -S|--style=FILE Location of the style file. Defaults to\n\ - '" DEFAULT_STYLE "'.\n\ - -k|--hstore Add tags without column to an additional hstore column.\n", - stdout); -#ifdef HAVE_LUA - (void)std::fputs("\ - --tag-transform-script=SCRIPT Specify a Lua script to handle tag\n\ - filtering and normalisation (pgsql output only).\n", - stdout); -#endif - (void)std::fputs("\ - -s|--slim Store temporary data in the database. This switch is\n\ - required if you want to update with --append later.\n\ - --drop Only with --slim: drop temporary tables after import\n\ - (no updates are possible).\n\ - -C|--cache=SIZE Use up to SIZE MB for caching nodes (default: 800).\n\ - -F|--flat-nodes=FILE Specifies the file to use to persistently store node\n\ - information in slim mode instead of in PostgreSQL.\n\ - This is a single large file (> 50GB). Only recommended\n\ - for full planet imports. Default is disabled.\n\ -\n\ -Database options:\n\ - -d|--database=DB The name of the PostgreSQL database to connect to or\n\ - a PostgreSQL conninfo string.\n\ - -U|--username=NAME PostgreSQL user name.\n\ - -W|--password Force password prompt.\n\ - -H|--host=HOST Database server host name or socket location.\n\ - -P|--port=PORT Database server port.\n", - stdout); - - if (verbose) { - (void)std::fputs("\n\ -Logging options:\n\ - --log-level=LEVEL Set log level ('debug', 'info' (default), 'warn',\n\ - or 'error').\n\ - --log-progress=VALUE Enable ('true') or disable ('false') progress\n\ - logging. If set to 'auto' osm2pgsql will enable progress\n\ - logging on the console and disable it if the output is\n\ - redirected to a file. Default: true.\n\ - --log-sql Enable logging of SQL commands for debugging.\n\ - --log-sql-data Enable logging of all data added to the database.\n\ - -v|--verbose Same as '--log-level=debug'.\n\ -\n\ -Input options:\n\ - -r|--input-reader=FORMAT Input format ('xml', 'pbf', 'o5m', or\n\ - 'auto' - autodetect format (default))\n\ - -b|--bbox=MINLON,MINLAT,MAXLON,MAXLAT Apply a bounding box filter on the\n\ - imported data, e.g. '--bbox -0.5,51.25,0.5,51.75'.\n\ -\n\ -Middle options:\n\ - -i|--tablespace-index=TBLSPC The name of the PostgreSQL tablespace where\n\ - all indexes will be created.\n\ - The following options allow more fine-grained control:\n\ - --tablespace-slim-data=TBLSPC Tablespace for slim mode tables.\n\ - --tablespace-slim-index=TBLSPC Tablespace for slim mode indexes.\n\ - (if unset, use db's default; -i is equivalent to setting\n\ - --tablespace-main-index and --tablespace-slim-index).\n\ - -p|--prefix=PREFIX Prefix for table names (default 'planet_osm')\n\ - --cache-strategy=STRATEGY Deprecated. Not used any more.\n\ - -x|--extra-attributes Include attributes (user name, user id, changeset\n\ - id, timestamp and version) for each object in the database.\n\ - --middle-schema=SCHEMA Schema to use for middle tables (default: none).\n\ - --middle-way-node-index-id-shift=SHIFT Set ID shift for bucket index.\n\ -\n\ -Pgsql output options:\n\ - -i|--tablespace-index=TBLSPC The name of the PostgreSQL tablespace where\n\ - all indexes will be created.\n\ - The following options allow more fine-grained control:\n\ - --tablespace-main-data=TBLSPC Tablespace for main tables.\n\ - --tablespace-main-index=TBLSPC Tablespace for main table indexes.\n\ - -l|--latlong Store data in degrees of latitude & longitude (WGS84).\n\ - -m|--merc Store data in web mercator (default).\n" -#ifdef HAVE_GENERIC_PROJ - " -E|--proj=SRID Use projection EPSG:SRID.\n" -#endif - "\ - -p|--prefix=PREFIX Prefix for table names (default 'planet_osm').\n\ - -x|--extra-attributes Include attributes (user name, user id, changeset\n\ - id, timestamp and version) for each object in the database.\n\ - --hstore-match-only Only keep objects that have a value in one of the\n\ - columns (default with --hstore is to keep all objects).\n\ - -j|--hstore-all Add all tags to an additional hstore (key/value) column.\n\ - -z|--hstore-column=NAME Add an additional hstore (key/value) column\n\ - containing all tags that start with the specified string,\n\ - eg '--hstore-column name:' will produce an extra hstore\n\ - column that contains all 'name:xx' tags.\n\ - --hstore-add-index Add index to hstore column.\n\ - -G|--multi-geometry Generate multi-geometry features in postgresql tables.\n\ - -K|--keep-coastlines Keep coastline data rather than filtering it out.\n\ - Default: discard objects tagged natural=coastline.\n\ - --output-pgsql-schema=SCHEMA Schema to use for pgsql output tables\n\ - (default: none).\n\ - --reproject-area Compute area column using web mercator coordinates.\n\ -\n\ -Expiry options:\n\ - -e|--expire-tiles=[MIN_ZOOM-]MAX_ZOOM Create a tile expiry list.\n\ - Zoom levels must be larger than 0 and smaller than 32.\n\ - -o|--expire-output=FILENAME Output filename for expired tiles list.\n\ - --expire-bbox-size=SIZE Max size for a polygon to expire the whole\n\ - polygon, not just the boundary.\n\ -\n\ -Advanced options:\n\ - -I|--disable-parallel-indexing Disable indexing all tables concurrently.\n\ - --number-processes=NUM Specifies the number of parallel processes used\n\ - for certain operations (default depends on number of CPUs).\n\ - --with-forward-dependencies=BOOL Propagate changes from nodes to ways\n\ - and node/way members to relations (Default: true).\n\ -", - stdout); - } else { - fmt::print( - stdout, - "\nRun '{} --help --verbose' (-h -v) for a full list of options.\n", - name); - } -} - -} // anonymous namespace - -static bool compare_prefix(std::string const &str, - std::string const &prefix) noexcept -{ - return std::strncmp(str.c_str(), prefix.c_str(), prefix.size()) == 0; -} - -std::string build_conninfo(database_options_t const &opt) -{ - if (compare_prefix(opt.db, "postgresql://") || - compare_prefix(opt.db, "postgres://")) { - return opt.db; - } - - util::string_joiner_t joiner{' '}; - joiner.add("fallback_application_name='osm2pgsql'"); - - if (std::strchr(opt.db.c_str(), '=') != nullptr) { - joiner.add(opt.db); - return joiner(); - } - - joiner.add("client_encoding='UTF8'"); - - if (!opt.db.empty()) { - joiner.add(fmt::format("dbname='{}'", opt.db)); - } - if (!opt.username.empty()) { - joiner.add(fmt::format("user='{}'", opt.username)); - } - if (!opt.password.empty()) { - joiner.add(fmt::format("password='{}'", opt.password)); - } - if (!opt.host.empty()) { - joiner.add(fmt::format("host='{}'", opt.host)); - } - if (!opt.port.empty()) { - joiner.add(fmt::format("port='{}'", opt.port)); - } - - return joiner(); -} - -options_t::options_t() -: num_procs(std::min(4U, std::thread::hardware_concurrency())) -{ - if (num_procs < 1) { - log_warn("Unable to detect number of hardware threads supported!" - " Using single thread."); - num_procs = 1; - } -} - -static osmium::Box parse_bbox_param(char const *arg) -{ - double minx = NAN; - double maxx = NAN; - double miny = NAN; - double maxy = NAN; - - int const n = sscanf(arg, "%lf,%lf,%lf,%lf", &minx, &miny, &maxx, &maxy); - if (n != 4) { - throw std::runtime_error{"Bounding box must be specified like: " - "minlon,minlat,maxlon,maxlat."}; - } - - if (maxx <= minx) { - throw std::runtime_error{ - "Bounding box failed due to maxlon <= minlon."}; - } - - if (maxy <= miny) { - throw std::runtime_error{ - "Bounding box failed due to maxlat <= minlat."}; - } - - log_debug("Applying bounding box: {},{} to {},{}", minx, miny, maxx, maxy); - - return osmium::Box{minx, miny, maxx, maxy}; -} - -static unsigned int parse_number_processes_param(char const *arg) -{ - int num = atoi(arg); - if (num < 1) { - log_warn("--number-processes must be at least 1. Using 1."); - num = 1; - } else if (num > 32) { - // The threads will open up database connections which will - // run out at some point. It depends on the number of tables - // how many connections there are. The number 32 is way beyond - // anything that will make sense here. - log_warn("--number-processes too large. Set to 32."); - num = 32; - } - - return static_cast(num); -} - -static void parse_expire_tiles_param(char const *arg, - uint32_t *expire_tiles_zoom_min, - uint32_t *expire_tiles_zoom) -{ - if (!arg || arg[0] == '-') { - throw std::runtime_error{"Missing argument for option --expire-tiles." - " Zoom levels must be positive."}; - } - - char *next_char = nullptr; - *expire_tiles_zoom_min = - static_cast(std::strtoul(arg, &next_char, 10)); - - if (*expire_tiles_zoom_min == 0) { - throw std::runtime_error{"Bad argument for option --expire-tiles." - " Minimum zoom level must be larger than 0."}; - } - - // The first character after the number is ignored because that is the - // separating hyphen. - if (*next_char == '-') { - ++next_char; - // Second number must not be negative because zoom levels must be - // positive. - if (next_char && *next_char != '-' && isdigit(*next_char)) { - char *after_maxzoom = nullptr; - *expire_tiles_zoom = static_cast( - std::strtoul(next_char, &after_maxzoom, 10)); - if (*expire_tiles_zoom == 0 || *after_maxzoom != '\0') { - throw std::runtime_error{"Invalid maximum zoom level" - " given for tile expiry."}; - } - } else { - throw std::runtime_error{ - "Invalid maximum zoom level given for tile expiry."}; - } - return; - } - - if (*next_char == '\0') { - // end of string, no second zoom level given - *expire_tiles_zoom = *expire_tiles_zoom_min; - return; - } - - throw std::runtime_error{"Minimum and maximum zoom level for" - " tile expiry must be separated by '-'."}; -} - -static void parse_log_level_param(char const *arg) -{ - if (std::strcmp(arg, "debug") == 0) { - get_logger().set_level(log_level::debug); - } else if (std::strcmp(arg, "info") == 0) { - get_logger().set_level(log_level::info); - } else if ((std::strcmp(arg, "warn") == 0) || - (std::strcmp(arg, "warning") == 0)) { - get_logger().set_level(log_level::warn); - } else if (std::strcmp(arg, "error") == 0) { - get_logger().set_level(log_level::error); - } else { - throw fmt_error("Unknown value for --log-level option: {}", arg); - } -} - -static void parse_log_progress_param(char const *arg) -{ - if (std::strcmp(arg, "true") == 0) { - get_logger().enable_progress(); - } else if (std::strcmp(arg, "false") == 0) { - get_logger().disable_progress(); - } else if (std::strcmp(arg, "auto") == 0) { - get_logger().auto_progress(); - } else { - throw fmt_error("Unknown value for --log-progress option: {}", arg); - } -} - -static bool parse_with_forward_dependencies_param(char const *arg) -{ - log_warn("The option --with-forward-dependencies is deprecated and will " - "soon be removed."); - - if (std::strcmp(arg, "false") == 0) { - return false; - } - - if (std::strcmp(arg, "true") == 0) { - return true; - } - - throw fmt_error("Unknown value for --with-forward-dependencies option: {}", - arg); -} - -static void print_version() -{ - fmt::print(stderr, "Build: {}\n", get_build_type()); - fmt::print(stderr, "Compiled using the following library versions:\n"); - fmt::print(stderr, "Libosmium {}\n", LIBOSMIUM_VERSION_STRING); - fmt::print(stderr, "Proj {}\n", get_proj_version()); -#ifdef HAVE_LUA -#ifdef HAVE_LUAJIT - fmt::print(stderr, "{} ({})\n", LUA_RELEASE, LUAJIT_VERSION); -#else - fmt::print(stderr, "{}\n", LUA_RELEASE); -#endif -#else - fmt::print(stderr, "Lua support not included\n"); -#endif -} - -options_t::options_t(int argc, char *argv[]) : options_t() -{ - // If there are no command line arguments at all, show help. - if (argc == 1) { - m_print_help = true; - long_usage(argv[0], false); - return; - } - - database_options_t database_options; - bool help_verbose = false; // Will be set when -v/--verbose is set - - int c = 0; - - //keep going while there are args left to handle - // note: optind would seem to need to be set to 1, but that gives valgrind - // errors - setting it to zero seems to work, though. see - // http://stackoverflow.com/questions/15179963/is-it-possible-to-repeat-getopt#15179990 - optind = 0; - // NOLINTNEXTLINE(concurrency-mt-unsafe) - while (-1 != (c = getopt_long(argc, argv, short_options, long_options, - nullptr))) { - - //handle the current arg - switch (c) { - case 'a': // --append - append = true; - break; - case 'b': // --bbox - bbox = parse_bbox_param(optarg); - break; - case 'c': // --create - create = true; - break; - case 'v': // --verbose - help_verbose = true; - get_logger().set_level(log_level::debug); - break; - case 's': // --slim - slim = true; - break; - case 'K': // --keep-coastlines - keep_coastlines = true; - break; - case 'l': // --latlong - projection = reprojection::create_projection(PROJ_LATLONG); - break; - case 'm': // --merc - projection = reprojection::create_projection(PROJ_SPHERE_MERC); - break; - case 'E': // --proj -#ifdef HAVE_GENERIC_PROJ - projection = reprojection::create_projection(atoi(optarg)); -#else - throw std::runtime_error{"Generic projections not available."}; -#endif - break; - case 'p': // --prefix - prefix = optarg; - check_identifier(prefix, "--prefix parameter"); - break; - case 'd': // --database - database_options.db = optarg; - break; - case 'C': // --cache - cache = atoi(optarg); - break; - case 'U': // --username - database_options.username = optarg; - break; - case 'W': // --password - pass_prompt = true; - break; - case 'H': // --host - database_options.host = optarg; - break; - case 'P': // --port - database_options.port = optarg; - break; - case 'S': // --style - style = optarg; - break; - case 'i': // --tablespace-index - tblsmain_index = optarg; - tblsslim_index = tblsmain_index; - break; - case 200: // --tablespace-slim-data - tblsslim_data = optarg; - break; - case 201: // --tablespace-slim-index - tblsslim_index = optarg; - break; - case 202: // --tablespace-main-data - tblsmain_data = optarg; - break; - case 203: // --tablespace-main-index - tblsmain_index = optarg; - break; - case 'e': // --expire-tiles - parse_expire_tiles_param(optarg, &expire_tiles_zoom_min, - &expire_tiles_zoom); - break; - case 'o': // --expire-output - expire_tiles_filename = optarg; - break; - case 214: // --expire-bbox-size - expire_tiles_max_bbox = atof(optarg); - break; - case 'O': // --output - output_backend = optarg; - break; - case 'x': // --extra-attributes - extra_attributes = true; - break; - case 'k': // --hstore - if (hstore_mode != hstore_column::none) { - throw std::runtime_error{"You can not specify both --hstore " - "(-k) and --hstore-all (-j)."}; - } - hstore_mode = hstore_column::norm; - break; - case 208: // --hstore-match-only - hstore_match_only = true; - break; - case 'j': // --hstore-all - if (hstore_mode != hstore_column::none) { - throw std::runtime_error{"You can not specify both --hstore " - "(-k) and --hstore-all (-j)."}; - } - hstore_mode = hstore_column::all; - break; - case 'z': // --hstore-column - hstore_columns.emplace_back(optarg); - break; - case 'G': // --multi-geometry - enable_multi = true; - break; - case 'r': // --input-reader - if (std::strcmp(optarg, "auto") != 0) { - input_format = optarg; - } - break; - case 'h': // --help - m_print_help = true; - break; - case 'I': // --disable-parallel-indexing - parallel_indexing = false; - break; - case 204: // -cache-strategy - log_warn("Deprecated option --cache-strategy ignored"); - break; - case 205: // --number-processes - num_procs = parse_number_processes_param(optarg); - break; - case 206: // --drop - droptemp = true; - break; - case 'F': // --flat-nodes - flat_node_file = optarg; - break; - case 211: // --hstore-add-index - enable_hstore_index = true; - break; - case 212: // --tag-transform-script - tag_transform_script = optarg; - break; - case 213: // --reproject-area - reproject_area = true; - break; - case 'V': // --version - print_version(); - std::exit(EXIT_SUCCESS); // NOLINT(concurrency-mt-unsafe) - break; - case 215: // --middle-schema - middle_dbschema = optarg; - check_identifier(middle_dbschema, "--middle-schema parameter"); - break; - case 216: // --output-pgsql-schema - output_dbschema = optarg; - check_identifier(output_dbschema, "--output-pgsql-schema parameter"); - break; - case 217: // --with-forward-dependencies=BOOL - with_forward_dependencies = - parse_with_forward_dependencies_param(optarg); - break; - case 300: // --middle-way-node-index-id-shift - way_node_index_id_shift = atoi(optarg); - break; - case 400: // --log-level=LEVEL - parse_log_level_param(optarg); - break; - case 401: // --log-progress=VALUE - parse_log_progress_param(optarg); - break; - case 402: // --log-sql - get_logger().enable_sql(); - break; - case 403: // --log-sql-data - get_logger().enable_sql_data(); - break; - case '?': - default: - throw std::runtime_error{"Usage error. Try 'osm2pgsql --help'."}; - } - } //end while - - //they were looking for usage info - if (m_print_help) { - long_usage(argv[0], help_verbose); - return; - } - - //we require some input files! - if (optind >= argc) { - throw std::runtime_error{ - "Missing input file(s). Try 'osm2pgsql --help'."}; - } - - //get the input files - while (optind < argc) { - input_files.emplace_back(argv[optind]); - ++optind; - } - - if (!projection) { - projection = reprojection::create_projection(PROJ_SPHERE_MERC); - } - - check_options(); - - if (pass_prompt) { - database_options.password = util::get_password(); - } - - conninfo = build_conninfo(database_options); -} - -void options_t::check_options() -{ - if (append && create) { - throw std::runtime_error{"--append and --create options can not be " - "used at the same time!"}; - } - - if (append && !slim) { - throw std::runtime_error{"--append can only be used with slim mode!"}; - } - - if (droptemp && !slim) { - throw std::runtime_error{"--drop only makes sense with --slim."}; - } - - if (hstore_mode == hstore_column::none && hstore_columns.empty() && - hstore_match_only) { - log_warn("--hstore-match-only only makes sense with --hstore, " - "--hstore-all, or --hstore-column; ignored."); - hstore_match_only = false; - } - - if (enable_hstore_index && hstore_mode == hstore_column::none && - hstore_columns.empty()) { - log_warn("--hstore-add-index only makes sense with hstore enabled; " - "ignored."); - enable_hstore_index = false; - } - - if (cache < 0) { - cache = 0; - log_warn("RAM cache cannot be negative. Using 0 instead."); - } - - if (cache == 0) { - if (!slim) { - throw std::runtime_error{ - "RAM node cache can only be disabled in slim mode."}; - } - if (flat_node_file.empty()) { - log_warn("RAM cache is disabled. This will likely slow down " - "processing a lot."); - } - } - - if (!slim && !flat_node_file.empty()) { - log_warn("Ignoring --flat-nodes/-F setting in non-slim mode"); - } - - // zoom level 31 is the technical limit because we use 32-bit integers for the x and y index of a tile ID - if (expire_tiles_zoom_min > 31) { - expire_tiles_zoom_min = 31; - log_warn("Minimum zoom level for tile expiry is too " - "large and has been set to 31."); - } - - if (expire_tiles_zoom > 31) { - expire_tiles_zoom = 31; - log_warn("Maximum zoom level for tile expiry is too " - "large and has been set to 31."); - } - - if (expire_tiles_zoom != 0 && projection->target_srs() != 3857) { - log_warn("Expire has been enabled (with -e or --expire-tiles) but " - "target SRS is not Mercator (EPSG:3857). Expire disabled!"); - expire_tiles_zoom = 0; - } - - if (output_backend == "flex" || output_backend == "gazetteer") { - if (style == DEFAULT_STYLE) { - throw std::runtime_error{ - "You have to set the config file with the -S|--style option."}; - } - } - - if (output_backend == "gazetteer") { - log_warn( - "The 'gazetteer' output is deprecated and will soon be removed."); - } -} diff -Nru osm2pgsql-1.8.1+ds/src/options.hpp osm2pgsql-1.9.0+ds/src/options.hpp --- osm2pgsql-1.8.1+ds/src/options.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/options.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -19,6 +19,13 @@ class reprojection; +enum class command_t +{ + help, + version, + process +}; + /// Variants for generation of hstore column enum class hstore_column : char { @@ -70,31 +77,14 @@ /** * Structure for storing command-line and other options */ -class options_t +struct options_t { -public: - /** - * Constructor setting default values for all options. Used for testing. - */ - options_t(); - - /** - * Constructor parsing the options from the command line. - */ - // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - options_t(int argc, char *argv[]); - - /** - * Return true if the main program should end directly after the option - * parsing. This is true when a help text was printed. - */ - bool early_return() const noexcept { - return m_print_help; - } + command_t command = command_t::process; std::string conninfo; ///< connection info for database std::string prefix{"planet_osm"}; ///< prefix for table names + bool prefix_is_set = false; /// Pg Tablespace to store indexes on main tables (no default TABLESPACE) std::string tblsmain_index{}; @@ -108,13 +98,16 @@ /// Pg Tablespace to store slim tables (no default TABLESPACE) std::string tblsslim_data{}; - /// Pg schema to store middle tables in, default none + /// Default Pg schema. + std::string dbschema{"public"}; + + /// Pg schema to store middle tables in. std::string middle_dbschema{}; - /// Pg schema to store output tables in, default none + /// Pg schema to store output tables in. std::string output_dbschema{}; - std::string style{DEFAULT_STYLE}; ///< style file to use + std::string style{}; ///< style file to use /// Name of the flat node file used. Empty if flat node file is not enabled. std::string flat_node_file{}; @@ -146,7 +139,7 @@ int cache = 800; ///< Memory usable for cache in MB - unsigned int num_procs; + unsigned int num_procs = 1; /** * How many bits should the node id be shifted for the way node index? @@ -156,6 +149,16 @@ */ uint8_t way_node_index_id_shift = 5; + /// Database format (0=unknown/no database middle, 1=legacy, 2=new) + uint8_t middle_database_format = 1; + + /** + * Should nodes (with tags) be stored in the middle? If no flat node file + * is used, nodes will always be stored. (Only works with the new middle + * database format.) + */ + bool middle_with_nodes = false; + /// add an additional hstore column with objects key/value pairs, and what type of hstore column hstore_column hstore_mode = hstore_column::none; @@ -185,14 +188,8 @@ bool create = false; bool pass_prompt = false; -private: - - bool m_print_help = false; - - /** - * Check input options for sanity - */ - void check_options(); -}; + bool output_backend_set = false; + bool style_set = false; +}; // struct options_t #endif // OSM2PGSQL_OPTIONS_HPP diff -Nru osm2pgsql-1.8.1+ds/src/ordered-index.hpp osm2pgsql-1.9.0+ds/src/ordered-index.hpp --- osm2pgsql-1.8.1+ds/src/ordered-index.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/ordered-index.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -64,7 +64,7 @@ * double their size until max_block_size is * reached. */ - explicit ordered_index_t(std::size_t initial_block_size = 1024 * 1024) + explicit ordered_index_t(std::size_t initial_block_size = 1024UL * 1024UL) : m_block_size(initial_block_size) {} @@ -185,7 +185,7 @@ std::pair get_internal(osmid_t id) const noexcept; - static constexpr std::size_t const max_block_size = 16 * 1024 * 1024; + static constexpr std::size_t const max_block_size = 16UL * 1024UL * 1024UL; std::vector m_ranges; std::size_t m_block_size; diff -Nru osm2pgsql-1.8.1+ds/src/osm2pgsql.cpp osm2pgsql-1.9.0+ds/src/osm2pgsql.cpp --- osm2pgsql-1.8.1+ds/src/osm2pgsql.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/osm2pgsql.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -7,6 +7,7 @@ * For a full list of authors see the git log. */ +#include "command-line-parser.hpp" #include "dependency-manager.hpp" #include "input.hpp" #include "logging.hpp" @@ -17,11 +18,14 @@ #include "pgsql.hpp" #include "pgsql-capabilities.hpp" #include "pgsql-helper.hpp" +#include "properties.hpp" #include "util.hpp" #include "version.hpp" #include +#include + #include #include #include @@ -40,7 +44,7 @@ } } -static void run(options_t const &options) +static file_info run(options_t const &options) { auto const files = prepare_input_files( options.input_files, options.input_format, options.append); @@ -68,32 +72,278 @@ // Processing: In this phase the input file(s) are read and parsed, // populating some of the tables. - process_files(files, &osmdata, options.append, - get_logger().show_progress()); + auto finfo = process_files(files, &osmdata, options.append, + get_logger().show_progress()); show_memory_usage(); // Process pending ways and relations. Cluster database tables and // create indexes. osmdata.stop(); + + return finfo; } -void check_db(options_t const &options) +static void check_db(options_t const &options) { pg_conn_t const db_connection{options.conninfo}; init_database_capabilities(db_connection); - // If we are in append mode and the middle nodes table isn't there, - // it probably means we used a flat node store when we created this - // database. Check for that and stop if it looks like we are missing - // the node location store option. - if (options.append && options.flat_node_file.empty()) { - if (!has_table(db_connection, options.middle_dbschema, - options.prefix + "_nodes")) { + check_schema(options.dbschema); + check_schema(options.middle_dbschema); + check_schema(options.output_dbschema); +} + +// This is called in "create" mode to store properties into the database. +static void store_properties(properties_t *properties, options_t const &options) +{ + properties->set_bool("attributes", options.extra_attributes); + + if (options.flat_node_file.empty()) { + properties->set_string("flat_node_file", ""); + } else { + properties->set_string( + "flat_node_file", + boost::filesystem::absolute( + boost::filesystem::path{options.flat_node_file}) + .string()); + } + + properties->set_string("prefix", options.prefix); + properties->set_bool("updatable", options.slim && !options.droptemp); + properties->set_string("version", get_osm2pgsql_short_version()); + properties->set_int("db_format", options.middle_database_format); + properties->set_string("output", options.output_backend); + + if (options.style.empty()) { + properties->set_string("style", ""); + } else { + properties->set_string( + "style", + boost::filesystem::absolute(boost::filesystem::path{options.style}) + .string()); + } + + properties->store(); +} + +static void store_data_properties(properties_t *properties, + file_info const &finfo) +{ + if (finfo.last_timestamp.valid()) { + auto const timestamp = finfo.last_timestamp.to_iso(); + properties->set_string("import_timestamp", timestamp); + properties->set_string("current_timestamp", timestamp); + } + + for (std::string const s : {"base_url", "sequence_number", "timestamp"}) { + auto const value = finfo.header.get("osmosis_replication_" + s); + if (!value.empty()) { + properties->set_string("replication_" + s, value); + } + } + + properties->store(); +} + +static void check_updatable(properties_t const &properties) +{ + if (properties.get_bool("updatable", false)) { + return; + } + + throw std::runtime_error{ + "This database is not updatable. To create an" + " updatable database use --slim (without --drop)."}; +} + +static void check_attributes(properties_t const &properties, options_t *options) +{ + bool const with_attributes = properties.get_bool("attributes", false); + + if (options->extra_attributes) { + if (!with_attributes) { throw std::runtime_error{ - "You seem to not have a nodes table. Did " - "you forget the --flat-nodes option?"}; + "Can not update with attributes (-x/--extra-attributes)" + " because original import was without attributes."}; + } + return; + } + + if (with_attributes) { + log_info("Updating with attributes (same as on import)."); + options->extra_attributes = true; + } +} + +static void check_and_update_flat_node_file(properties_t *properties, + options_t *options) +{ + auto const flat_node_file_from_import = + properties->get_string("flat_node_file", ""); + if (options->flat_node_file.empty()) { + if (flat_node_file_from_import.empty()) { + log_info("Not using flat node file (same as on import)."); + } else { + options->flat_node_file = flat_node_file_from_import; + log_info("Using flat node file '{}' (same as on import).", + flat_node_file_from_import); + } + } else { + const auto absolute_path = + boost::filesystem::absolute( + boost::filesystem::path{options->flat_node_file}) + .string(); + + if (flat_node_file_from_import.empty()) { + throw fmt_error("Database was imported without flat node file. Can" + " not use flat node file '{}' now.", + options->flat_node_file); + } + + if (absolute_path == flat_node_file_from_import) { + log_info("Using flat node file '{}' (same as on import).", + flat_node_file_from_import); + } else { + log_info( + "Using the flat node file you specified on the command line" + " ('{}') instead of the one used on import ('{}').", + absolute_path, flat_node_file_from_import); + properties->set_string("flat_node_file", absolute_path, true); + } + } +} + +static void check_prefix(properties_t const &properties, options_t *options) +{ + auto const prefix = properties.get_string("prefix", "planet_osm"); + if (!options->prefix_is_set) { + log_info("Using prefix '{}' (same as on import).", prefix); + options->prefix = prefix; + return; + } + + if (prefix != options->prefix) { + throw fmt_error("Different prefix specified on command line ('{}')" + " then used on import ('{}').", + options->prefix, prefix); + } +} + +static void check_db_format(properties_t const &properties, options_t *options) +{ + auto const format = properties.get_int("db_format", -1); + + if (format == -1) { + // db_format not set, so this is a legacy import + return; + } + + if (format == 0) { + throw std::runtime_error{ + "This database is not updatable (db_format=0)."}; + } + + if (format < -1 || format > 2) { + throw fmt_error("Unknown db_format '{}' in properties.", format); + } + + options->middle_database_format = format; +} + +static void check_output(properties_t const &properties, options_t *options) +{ + auto const output = properties.get_string("output", "pgsql"); + + if (!options->output_backend_set) { + options->output_backend = output; + log_info("Using output '{}' (same as on import).", output); + return; + } + + if (options->output_backend == output) { + return; + } + + throw fmt_error("Different output specified on command line ('{}')" + " then used on import ('{}').", + options->output_backend, output); +} + +static void check_and_update_style_file(properties_t *properties, + options_t *options) +{ + auto const style_file_from_import = properties->get_string("style", ""); + + if (options->style.empty()) { + log_info("Using style file '{}' (same as on import).", + style_file_from_import); + options->style = style_file_from_import; + return; + } + + if (style_file_from_import.empty()) { + throw std::runtime_error{"Style file from import is empty!?"}; + } + + const auto absolute_path = + boost::filesystem::absolute(boost::filesystem::path{options->style}) + .string(); + + if (absolute_path == style_file_from_import) { + log_info("Using style file '{}' (same as on import).", + style_file_from_import); + return; + } + + log_info("Using the style file you specified on the command line" + " ('{}') instead of the one used on import ('{}').", + absolute_path, style_file_from_import); + properties->set_string("style", absolute_path, true); +} + +// This is called in "append" mode to check that the command line options are +// compatible with the properties stored in the database. +static void check_and_update_properties(properties_t *properties, + options_t *options) +{ + check_updatable(*properties); + check_attributes(*properties, options); + check_and_update_flat_node_file(properties, options); + check_prefix(*properties, options); + check_db_format(*properties, options); + check_output(*properties, options); + check_and_update_style_file(properties, options); +} + +// If we are in append mode and the middle nodes table isn't there, it probably +// means we used a flat node store when we created this database. Check for +// that and stop if it looks like we are missing the node location store +// option. (This function is only used in legacy systems which don't have the +// properties stored in the database.) +static void check_for_nodes_table(options_t const &options) +{ + if (!options.flat_node_file.empty()) { + return; + } + + if (!has_table(options.middle_dbschema, options.prefix + "_nodes")) { + throw std::runtime_error{"You seem to not have a nodes table. Did " + "you forget the --flat-nodes option?"}; + } +} + +static void check_and_set_style(options_t *options) +{ + if (!options->style_set) { + if (options->output_backend == "flex" || + options->output_backend == "gazetteer") { + throw std::runtime_error{"You have to set the config file " + "with the -S|--style option."}; + } + if (options->output_backend == "pgsql") { + options->style = DEFAULT_STYLE; } } } @@ -103,8 +353,15 @@ try { log_info("osm2pgsql version {}", get_osm2pgsql_version()); - options_t const options{argc, argv}; - if (options.early_return()) { + auto options = parse_command_line(argc, argv); + + if (options.command == command_t::help) { + // Already handled inside parse_command_line() + return 0; + } + + if (options.command == command_t::version) { + print_version(); return 0; } @@ -112,7 +369,34 @@ check_db(options); - run(options); + properties_t properties{options.conninfo, options.middle_dbschema}; + if (options.append) { + if (properties.load()) { + check_and_update_properties(&properties, &options); + } else { + check_and_set_style(&options); + check_for_nodes_table(options); + } + + auto const finfo = run(options); + + if (finfo.last_timestamp.valid()) { + auto const current_timestamp = + properties.get_string("current_timestamp", ""); + + if (current_timestamp.empty() || + (finfo.last_timestamp > + osmium::Timestamp{current_timestamp})) { + properties.set_string("current_timestamp", + finfo.last_timestamp.to_iso(), true); + } + } + } else { + check_and_set_style(&options); + store_properties(&properties, options); + auto const finfo = run(options); + store_data_properties(&properties, finfo); + } show_memory_usage(); log_info("osm2pgsql took {} overall.", diff -Nru osm2pgsql-1.8.1+ds/src/osmdata.cpp osm2pgsql-1.9.0+ds/src/osmdata.cpp --- osm2pgsql-1.8.1+ds/src/osmdata.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/osmdata.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -78,6 +78,9 @@ { m_mid->after_nodes(); m_output->after_nodes(); + if (m_append) { + m_dependency_manager->after_nodes(); + } } void osmdata_t::way(osmium::Way &way) @@ -106,6 +109,9 @@ { m_mid->after_ways(); m_output->after_ways(); + if (m_append) { + m_dependency_manager->after_ways(); + } } void osmdata_t::relation(osmium::Relation const &rel) @@ -135,12 +141,19 @@ } else { m_output->relation_delete(rel.id()); } + m_dependency_manager->relation_changed(rel.id()); } else if (has_tags_or_attrs) { m_output->relation_add(rel); } } -void osmdata_t::after_relations() { m_mid->after_relations(); } +void osmdata_t::after_relations() +{ + m_mid->after_relations(); + if (m_append) { + m_dependency_manager->after_relations(); + } +} void osmdata_t::start() const { @@ -348,10 +361,13 @@ } // stage 1c processing: mark parent relations of marked objects as changed - for (auto const id : m_output->get_marked_way_ids()) { - m_dependency_manager->way_changed(id); + auto marked_ways = m_output->get_marked_way_ids(); + if (marked_ways.empty()) { + return; } + m_dependency_manager->mark_parent_relations_as_pending(marked_ways); + // process parent relations of marked ways if (m_dependency_manager->has_pending()) { proc.process_relations_stage1c( diff -Nru osm2pgsql-1.8.1+ds/src/osmdata.hpp osm2pgsql-1.9.0+ds/src/osmdata.hpp --- osm2pgsql-1.8.1+ds/src/osmdata.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/osmdata.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -13,8 +13,6 @@ /** * \file * - * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql). - * * It contains the osmdata_t class. */ @@ -29,8 +27,8 @@ #include "osmtypes.hpp" class middle_t; -class options_t; class output_t; +struct options_t; /** * This class guides the processing of the OSM data through its multiple diff -Nru osm2pgsql-1.8.1+ds/src/osmtypes.hpp osm2pgsql-1.9.0+ds/src/osmtypes.hpp --- osm2pgsql-1.8.1+ds/src/osmtypes.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/osmtypes.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -13,8 +13,6 @@ /** * \file * - * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql). - * * In this file some basic (OSM) data types are defined. */ diff -Nru osm2pgsql-1.8.1+ds/src/output-flex.cpp osm2pgsql-1.9.0+ds/src/output-flex.cpp --- osm2pgsql-1.8.1+ds/src/output-flex.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/output-flex.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -8,11 +8,14 @@ */ #include "db-copy.hpp" +#include "debug-output.hpp" +#include "expire-output.hpp" #include "expire-tiles.hpp" #include "flex-index.hpp" #include "flex-lua-geom.hpp" #include "flex-lua-index.hpp" #include "flex-lua-table.hpp" +#include "flex-lua-expire-output.hpp" #include "flex-write.hpp" #include "format.hpp" #include "geom-from-osm.hpp" @@ -20,6 +23,7 @@ #include "geom-transform.hpp" #include "logging.hpp" #include "lua-init.hpp" +#include "lua-setup.hpp" #include "lua-utils.hpp" #include "middle.hpp" #include "options.hpp" @@ -31,17 +35,10 @@ #include "thread-pool.hpp" #include "util.hpp" #include "version.hpp" +#include "wkb.hpp" #include -extern "C" -{ -#include -#include -} - -#include - #include #include #include @@ -51,9 +48,6 @@ #include #include -// Mutex used to coordinate access to Lua code -static std::mutex lua_mutex; - // Lua can't call functions on C++ objects directly. This macro defines simple // C "trampoline" functions which are called from Lua which get the current // context (the output_flex_t object) and call the respective function on the @@ -75,6 +69,7 @@ } TRAMPOLINE(app_define_table, define_table) +TRAMPOLINE(app_define_expire_output, define_expire_output) TRAMPOLINE(app_get_bbox, get_bbox) TRAMPOLINE(app_as_point, as_point) @@ -93,6 +88,13 @@ TRAMPOLINE(table_columns, columns) TRAMPOLINE(table_tostring, __tostring) +TRAMPOLINE(expire_output_minzoom, minzoom) +TRAMPOLINE(expire_output_maxzoom, maxzoom) +TRAMPOLINE(expire_output_filename, filename) +TRAMPOLINE(expire_output_schema, schema) +TRAMPOLINE(expire_output_table, table) +TRAMPOLINE(expire_output_tostring, __tostring) + static char const *const osm2pgsql_object_metatable = "osm2pgsql.object_metatable"; @@ -142,63 +144,19 @@ if (with_attributes) { if (object.version() != 0U) { luaX_add_table_int(lua_state, "version", object.version()); - } else { - // This is a workaround, because the middle will give us the - // attributes as pseudo-tags. - char const *const val = object.tags()["osm_version"]; - if (val) { - luaX_add_table_int(lua_state, "version", - osmium::string_to_object_version(val)); - } } - if (object.timestamp().valid()) { luaX_add_table_int(lua_state, "timestamp", object.timestamp().seconds_since_epoch()); - } else { - // This is a workaround, because the middle will give us the - // attributes as pseudo-tags. - char const *const val = object.tags()["osm_timestamp"]; - if (val) { - auto const timestamp = osmium::Timestamp{val}; - luaX_add_table_int(lua_state, "timestamp", - timestamp.seconds_since_epoch()); - } } - if (object.changeset() != 0U) { luaX_add_table_int(lua_state, "changeset", object.changeset()); - } else { - char const *const val = object.tags()["osm_changeset"]; - // This is a workaround, because the middle will give us the - // attributes as pseudo-tags. - if (val) { - luaX_add_table_int(lua_state, "changeset", - osmium::string_to_changeset_id(val)); - } } - if (object.uid() != 0U) { luaX_add_table_int(lua_state, "uid", object.uid()); - } else { - // This is a workaround, because the middle will give us the - // attributes as pseudo-tags. - char const *const val = object.tags()["osm_uid"]; - if (val) { - luaX_add_table_int(lua_state, "uid", - osmium::string_to_uid(val)); - } } - if (object.user()[0] != '\0') { luaX_add_table_str(lua_state, "user", object.user()); - } else { - // This is a workaround, because the middle will give us the - // attributes as pseudo-tags. - char const *const val = object.tags()["osm_user"]; - if (val) { - luaX_add_table_str(lua_state, "user", val); - } } } @@ -443,42 +401,69 @@ " main Lua code, not in any of the callbacks."}; } - return setup_flex_table(lua_state(), m_tables.get(), - get_options()->slim && !get_options()->droptemp); + return setup_flex_table(lua_state(), m_tables.get(), m_expire_outputs.get(), + get_options()->dbschema, + get_options()->slim && !get_options()->droptemp, + get_options()->append); } -// Check that the first element on the Lua stack is an osm2pgsql.Table -// parameter and return its internal table index. -static std::size_t table_idx_from_param(lua_State *lua_state) +int output_flex_t::app_define_expire_output() { + if (m_calling_context != calling_context::main) { + throw std::runtime_error{ + "Expire outputs have to be defined in the" + " main Lua code, not in any of the callbacks."}; + } + + return setup_flex_expire_output(lua_state(), get_options()->dbschema, + m_expire_outputs.get()); +} + +// Check that the first element on the Lua stack is a "type_name" +// parameter and return its internal index. +static std::size_t idx_from_param(lua_State *lua_state, char const *type_name) +{ + assert(lua_gettop(lua_state) >= 1); + void const *const user_data = lua_touserdata(lua_state, 1); if (user_data == nullptr || !lua_getmetatable(lua_state, 1)) { - throw std::runtime_error{ - "First parameter must be of type osm2pgsql.Table."}; + throw fmt_error("First parameter must be of type {}.", type_name); } - luaL_getmetatable(lua_state, osm2pgsql_table_name); + luaL_getmetatable(lua_state, type_name); if (!lua_rawequal(lua_state, -1, -2)) { - throw std::runtime_error{ - "First parameter must be of type osm2pgsql.Table."}; + throw fmt_error("First parameter must be of type {}.", type_name); } - lua_pop(lua_state, 2); + lua_pop(lua_state, 2); // remove the two metatables return *static_cast(user_data); } -// Get the flex table that is as first parameter on the Lua stack. +template +static typename CONTAINER::value_type const & +get_from_idx_param(lua_State *lua_state, CONTAINER *container, + char const *type_name) +{ + if (lua_gettop(lua_state) != 1) { + throw fmt_error("Need exactly one parameter of type {}.", type_name); + } + + auto const &item = container->at(idx_from_param(lua_state, type_name)); + lua_remove(lua_state, 1); + return item; +} + flex_table_t const &output_flex_t::get_table_from_param() { - if (lua_gettop(lua_state()) != 1) { - throw std::runtime_error{ - "Need exactly one parameter of type osm2pgsql.Table."}; - } + return get_from_idx_param(lua_state(), m_tables.get(), + osm2pgsql_table_name); +} - auto const &table = m_tables->at(table_idx_from_param(lua_state())); - lua_remove(lua_state(), 1); - return table; +expire_output_t const &output_flex_t::get_expire_output_from_param() +{ + return get_from_idx_param(lua_state(), m_expire_outputs.get(), + osm2pgsql_expire_output_name); } int output_flex_t::table_tostring() @@ -588,8 +573,9 @@ "Need two parameters: The osm2pgsql.Table and the row data."}; } - auto &table_connection = - m_table_connections.at(table_idx_from_param(lua_state())); + auto &table_connection = m_table_connections.at( + idx_from_param(lua_state(), osm2pgsql_table_name)); + auto const &table = table_connection.table(); // If there is a second parameter, it must be a Lua table. @@ -665,8 +651,8 @@ } // The first parameter is the table object. - auto &table_connection = - m_table_connections.at(table_idx_from_param(lua_state())); + auto &table_connection = m_table_connections.at( + idx_from_param(lua_state(), osm2pgsql_table_name)); // The second parameter must be a Lua table with the contents for the // fields. @@ -690,8 +676,8 @@ } else if (column.type() == table_column_type::id_num) { copy_mgr->add_column(id); } else { - flex_write_column(lua_state(), copy_mgr, column, &m_expire, - m_expire_config); + flex_write_column(lua_state(), copy_mgr, column, + &m_expire_tiles); } } table_connection.increment_insert_counter(); @@ -806,7 +792,7 @@ if (!table.has_geom_column()) { flex_write_row(lua_state(), table_connection, object.type(), id, {}, 0, - &m_expire, m_expire_config); + &m_expire_tiles); return; } @@ -844,15 +830,74 @@ auto const geoms = geom::split_multi(std::move(geom), split_multi); for (auto const &sgeom : geoms) { - m_expire.from_geometry_if_3857(sgeom, m_expire_config); + table.geom_column().do_expire(sgeom, &m_expire_tiles); flex_write_row(lua_state(), table_connection, object.type(), id, sgeom, - table.geom_column().srid(), &m_expire, m_expire_config); + table.geom_column().srid(), &m_expire_tiles); + table_connection->increment_insert_counter(); } } +int output_flex_t::expire_output_tostring() +{ + auto const &expire_output = get_expire_output_from_param(); + + std::string const str = + fmt::format("osm2pgsql.ExpireOutput[minzoom={},maxzoom={},filename={}," + "schema={},table={}]", + expire_output.minzoom(), expire_output.maxzoom(), + expire_output.filename(), expire_output.schema(), + expire_output.table()); + lua_pushstring(lua_state(), str.c_str()); + + return 1; +} + +int output_flex_t::expire_output_minzoom() +{ + auto const &expire_output = get_expire_output_from_param(); + + lua_pushinteger(lua_state(), expire_output.minzoom()); + return 1; +} + +int output_flex_t::expire_output_maxzoom() +{ + auto const &expire_output = get_expire_output_from_param(); + + lua_pushinteger(lua_state(), expire_output.maxzoom()); + return 1; +} + +int output_flex_t::expire_output_filename() +{ + auto const &expire_output = get_expire_output_from_param(); + + lua_pushstring(lua_state(), expire_output.filename().c_str()); + return 1; +} + +int output_flex_t::expire_output_schema() +{ + auto const &expire_output = get_expire_output_from_param(); + + lua_pushstring(lua_state(), expire_output.schema().c_str()); + return 1; +} + +int output_flex_t::expire_output_table() +{ + auto const &expire_output = get_expire_output_from_param(); + + lua_pushstring(lua_state(), expire_output.table().c_str()); + return 1; +} + void output_flex_t::call_lua_function(prepared_lua_function_t func, osmium::OSMObject const &object) { + static std::mutex lua_mutex; + std::lock_guard const guard{lua_mutex}; + m_calling_context = func.context(); lua_pushvalue(lua_state(), func.index()); // the function to call @@ -869,13 +914,6 @@ m_calling_context = calling_context::main; } -void output_flex_t::get_mutex_and_call_lua_function( - prepared_lua_function_t func, osmium::OSMObject const &object) -{ - std::lock_guard const guard{lua_mutex}; - call_lua_function(func, object); -} - void output_flex_t::pending_way(osmid_t id) { if (!m_process_way) { @@ -888,7 +926,7 @@ way_delete(id); - get_mutex_and_call_lua_function(m_process_way, m_way_cache.get()); + call_lua_function(m_process_way, m_way_cache.get()); } void output_flex_t::select_relation_members() @@ -897,7 +935,6 @@ return; } - std::lock_guard const guard{lua_mutex}; call_lua_function(m_select_relation_members, m_relation_cache.get()); // If the function returned nil there is nothing to be marked. @@ -977,8 +1014,7 @@ delete_from_tables(osmium::item_type::relation, id); if (m_process_relation) { - get_mutex_and_call_lua_function(m_process_relation, - m_relation_cache.get()); + call_lua_function(m_process_relation, m_relation_cache.get()); } } @@ -993,7 +1029,7 @@ } m_disable_add_row = true; - get_mutex_and_call_lua_function(m_process_relation, m_relation_cache.get()); + call_lua_function(m_process_relation, m_relation_cache.get()); m_disable_add_row = false; } @@ -1027,19 +1063,37 @@ })); } - if (get_options()->expire_tiles_zoom_min > 0) { - auto const count = output_tiles_to_file( - m_expire.get_tiles(), get_options()->expire_tiles_zoom_min, - get_options()->expire_tiles_zoom, - get_options()->expire_tiles_filename); - log_info("Wrote {} entries to expired tiles list", count); + assert(m_expire_outputs->size() == m_expire_tiles.size()); + for (std::size_t i = 0; i < m_expire_outputs->size(); ++i) { + if (!m_expire_tiles[i].empty()) { + auto const &eo = (*m_expire_outputs)[i]; + + std::size_t const count = eo.output(m_expire_tiles[i].get_tiles(), + get_options()->conninfo); + + log_info("Wrote {} entries to expire output [{}].", count, i); + } } } void output_flex_t::wait() { + std::exception_ptr eptr; + flex_table_t const *table_with_error = nullptr; + for (auto &table : m_table_connections) { - table.task_wait(); + try { + table.task_wait(); + } catch (...) { + eptr = std::current_exception(); + table_with_error = &table.table(); + } + } + + if (eptr) { + log_error("Error while doing postprocessing on table '{}':", + table_with_error->name()); + std::rethrow_exception(eptr); } } @@ -1050,7 +1104,7 @@ } m_context_node = &node; - get_mutex_and_call_lua_function(m_process_node, node); + call_lua_function(m_process_node, node); m_context_node = nullptr; } @@ -1063,7 +1117,7 @@ } m_way_cache.init(way); - get_mutex_and_call_lua_function(m_process_way, m_way_cache.get()); + call_lua_function(m_process_way, m_way_cache.get()); } void output_flex_t::relation_add(osmium::Relation const &relation) @@ -1074,20 +1128,30 @@ m_relation_cache.init(relation); select_relation_members(); - get_mutex_and_call_lua_function(m_process_relation, relation); + call_lua_function(m_process_relation, relation); } void output_flex_t::delete_from_table(table_connection_t *table_connection, osmium::item_type type, osmid_t osm_id) { assert(table_connection); - auto const &table = table_connection->table(); - auto const id = table.map_id(type, osm_id); + auto const id = table_connection->table().map_id(type, osm_id); - if (m_expire.enabled() && table.has_geom_column() && - table.geom_column().srid() == 3857) { - auto const result = table_connection->get_geom_by_id(type, id); - expire_from_result(&m_expire, result, m_expire_config); + if (table_connection->table().has_columns_with_expire()) { + auto const result = table_connection->get_geoms_by_id(type, id); + auto const num_tuples = result.num_tuples(); + if (num_tuples > 0) { + int col = 0; + for (auto const &column : table_connection->table()) { + if (column.has_expire()) { + for (int i = 0; i < num_tuples; ++i) { + auto const geom = ewkb_to_geom(result.get(i, col)); + column.do_expire(geom, &m_expire_tiles); + } + ++col; + } + } + } } table_connection->delete_rows_with(type, id); @@ -1147,15 +1211,32 @@ } } +static void +create_expire_tables(std::vector const &expire_outputs, + std::string const &conninfo) +{ + if (std::all_of(expire_outputs.begin(), expire_outputs.end(), + [](auto const &expire_output) { + return expire_output.table().empty(); + })) { + return; + } + + pg_conn_t const connection{conninfo}; + for (auto const &expire_output : expire_outputs) { + if (!expire_output.table().empty()) { + expire_output.create_output_table(connection); + } + } +} + output_flex_t::output_flex_t(output_flex_t const *other, std::shared_ptr mid, std::shared_ptr copy_thread) : output_t(other, std::move(mid)), m_tables(other->m_tables), + m_expire_outputs(other->m_expire_outputs), m_stage2_way_ids(other->m_stage2_way_ids), m_copy_thread(std::move(copy_thread)), m_lua_state(other->m_lua_state), - m_expire_config(other->m_expire_config), - m_expire(other->get_options()->expire_tiles_zoom, - other->get_options()->projection), m_process_node(other->m_process_node), m_process_way(other->m_process_way), m_process_relation(other->m_process_relation), m_select_relation_members(other->m_select_relation_members) @@ -1165,6 +1246,11 @@ tc.connect(get_options()->conninfo); tc.prepare(); } + + for (auto &expire_output : *m_expire_outputs) { + m_expire_tiles.emplace_back(expire_output.maxzoom(), + reprojection::create_projection(3857)); + } } std::shared_ptr @@ -1178,14 +1264,8 @@ std::shared_ptr thread_pool, options_t const &options) : output_t(mid, std::move(thread_pool), options), - m_copy_thread(std::make_shared(options.conninfo)), - m_expire(options.expire_tiles_zoom, options.projection) + m_copy_thread(std::make_shared(options.conninfo)) { - m_expire_config.full_area_limit = get_options()->expire_tiles_max_bbox; - if (get_options()->expire_tiles_max_bbox > 0.0) { - m_expire_config.mode = expire_mode::hybrid; - } - init_lua(options.style); // If the osm2pgsql.select_relation_members() Lua function is defined @@ -1200,32 +1280,40 @@ "No tables defined in Lua config. Nothing to do!"}; } - log_debug("Tables:"); - for (auto const &table : *m_tables) { - log_debug("- TABLE {}", qualified_name(table.schema(), table.name())); - log_debug(" - columns:"); - for (auto const &column : table) { - log_debug(R"( - "{}" {} ({}) not_null={} create_only={})", - column.name(), column.type_name(), column.sql_type_name(), - column.not_null(), column.create_only()); - } - log_debug(" - data_tablespace={}", table.data_tablespace()); - log_debug(" - index_tablespace={}", table.index_tablespace()); - log_debug(" - cluster={}", table.cluster_by_geom()); - for (auto const &index : table.indexes()) { - log_debug(" - INDEX USING {}", index.method()); - log_debug(" - column={}", index.columns()); - log_debug(" - expression={}", index.expression()); - log_debug(" - include={}", index.include_columns()); - log_debug(" - tablespace={}", index.tablespace()); - log_debug(" - unique={}", index.is_unique()); - log_debug(" - where={}", index.where_condition()); + // For backwards compatibility we add a "default" expire output to all + // tables when the relevant command line options are used. + if (options.append && options.expire_tiles_zoom) { + auto &eo = m_expire_outputs->emplace_back(); + eo.set_filename(options.expire_tiles_filename); + eo.set_minzoom(options.expire_tiles_zoom_min); + eo.set_maxzoom(options.expire_tiles_zoom); + + for (auto &table : *m_tables) { + if (table.has_geom_column() && table.geom_column().srid() == 3857) { + expire_config_t config{}; + config.expire_output = m_expire_outputs->size() - 1; + if (options.expire_tiles_max_bbox > 0.0) { + config.mode = expire_mode::hybrid; + config.full_area_limit = options.expire_tiles_max_bbox; + } + table.geom_column().add_expire(config); + } } } + write_expire_output_list_to_debug_log(*m_expire_outputs); + write_table_list_to_debug_log(*m_tables); + for (auto &table : *m_tables) { m_table_connections.emplace_back(&table, m_copy_thread); } + + for (auto &expire_output : *m_expire_outputs) { + m_expire_tiles.emplace_back(expire_output.maxzoom(), + reprojection::create_projection(3857)); + } + + create_expire_tables(*m_expire_outputs, get_options()->conninfo); } /** @@ -1252,6 +1340,40 @@ luaX_add_table_func(lua_state, "schema", lua_trampoline_table_schema); luaX_add_table_func(lua_state, "cluster", lua_trampoline_table_cluster); luaX_add_table_func(lua_state, "columns", lua_trampoline_table_columns); + + lua_pop(lua_state, 2); +} + +/** + * Define the osm2pgsql.ExpireOutput class/metatable. + */ +static void init_expire_output_class(lua_State *lua_state) +{ + lua_getglobal(lua_state, "osm2pgsql"); + if (luaL_newmetatable(lua_state, osm2pgsql_expire_output_name) != 1) { + throw std::runtime_error{"Internal error: Lua newmetatable failed."}; + } + lua_pushvalue(lua_state, -1); // Copy of new metatable + + // Add metatable as osm2pgsql.ExpireOutput so we can access it from Lua + lua_setfield(lua_state, -3, "ExpireOutput"); + + // Now add functions to metatable + lua_pushvalue(lua_state, -1); + lua_setfield(lua_state, -2, "__index"); + luaX_add_table_func(lua_state, "__tostring", + lua_trampoline_expire_output_tostring); + luaX_add_table_func(lua_state, "minzoom", + lua_trampoline_expire_output_minzoom); + luaX_add_table_func(lua_state, "maxzoom", + lua_trampoline_expire_output_maxzoom); + luaX_add_table_func(lua_state, "filename", + lua_trampoline_expire_output_filename); + luaX_add_table_func(lua_state, "schema", + lua_trampoline_expire_output_schema); + luaX_add_table_func(lua_state, "table", lua_trampoline_expire_output_table); + + lua_pop(lua_state, 2); } void output_flex_t::init_lua(std::string const &filename) @@ -1259,30 +1381,18 @@ m_lua_state.reset(luaL_newstate(), [](lua_State *state) { lua_close(state); }); - // Set up global lua libs - luaL_openlibs(lua_state()); + setup_lua_environment(lua_state(), filename, get_options()->append); - // Set up global "osm2pgsql" object - lua_newtable(lua_state()); - - luaX_add_table_str(lua_state(), "version", get_osm2pgsql_short_version()); - luaX_add_table_str(lua_state(), "mode", - get_options()->append ? "append" : "create"); luaX_add_table_int(lua_state(), "stage", 1); - std::string dir_path = - boost::filesystem::path{filename}.parent_path().string(); - if (!dir_path.empty()) { - dir_path += boost::filesystem::path::preferred_separator; - } - luaX_add_table_str(lua_state(), "config_dir", dir_path.c_str()); - luaX_add_table_func(lua_state(), "define_table", lua_trampoline_app_define_table); - lua_setglobal(lua_state(), "osm2pgsql"); + luaX_add_table_func(lua_state(), "define_expire_output", + lua_trampoline_app_define_expire_output); init_table_class(lua_state()); + init_expire_output_class(lua_state()); // Clean up stack lua_settop(lua_state(), 0); @@ -1408,7 +1518,7 @@ } way_delete(id); if (m_process_way) { - get_mutex_and_call_lua_function(m_process_way, m_way_cache.get()); + call_lua_function(m_process_way, m_way_cache.get()); } } @@ -1418,8 +1528,9 @@ void output_flex_t::merge_expire_trees(output_t *other) { - auto *opgsql = dynamic_cast(other); - if (opgsql) { - m_expire.merge_and_destroy(&opgsql->m_expire); + auto *const opgsql = dynamic_cast(other); + assert(m_expire_tiles.size() == opgsql->m_expire_tiles.size()); + for (std::size_t i = 0; i < m_expire_tiles.size(); ++i) { + m_expire_tiles[i].merge_and_destroy(&opgsql->m_expire_tiles[i]); } } diff -Nru osm2pgsql-1.8.1+ds/src/output-flex.hpp osm2pgsql-1.9.0+ds/src/output-flex.hpp --- osm2pgsql-1.8.1+ds/src/output-flex.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/output-flex.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -11,6 +11,7 @@ */ #include "expire-config.hpp" +#include "expire-output.hpp" #include "expire-tiles.hpp" #include "flex-table-column.hpp" #include "flex-table.hpp" @@ -20,10 +21,7 @@ #include #include -extern "C" -{ -#include -} +#include #include #include @@ -34,8 +32,8 @@ class db_copy_thread_t; class db_deleter_by_type_and_id_t; class geom_transform_t; -class options_t; class thread_pool_t; +struct options_t; using idset_t = osmium::index::IdSetSmall; @@ -162,6 +160,7 @@ int app_as_geometrycollection(); int app_define_table(); + int app_define_expire_output(); int app_get_bbox(); int app_mark_way(); @@ -173,25 +172,33 @@ int table_cluster(); int table_columns(); + int expire_output_tostring(); + int expire_output_name(); + int expire_output_minzoom(); + int expire_output_maxzoom(); + int expire_output_filename(); + int expire_output_schema(); + int expire_output_table(); + private: void select_relation_members(); /** * Call a Lua function that was "prepared" earlier with the OSMObject - * as its only parameter. + * as its only parameter. Uses a mutex internally to make access to the + * Lua environment thread safe. */ void call_lua_function(prepared_lua_function_t func, osmium::OSMObject const &object); - /// Aquire the lua_mutex and the call `call_lua_function()`. - void get_mutex_and_call_lua_function(prepared_lua_function_t func, - osmium::OSMObject const &object); - void init_lua(std::string const &filename); // Get the flex table that is as first parameter on the Lua stack. flex_table_t const &get_table_from_param(); + // Get the expire output that is as first parameter on the Lua stack. + expire_output_t const &get_expire_output_from_param(); + void check_context_and_state(char const *name, char const *context, bool condition); @@ -273,6 +280,9 @@ std::shared_ptr> m_tables = std::make_shared>(); + std::shared_ptr> m_expire_outputs = + std::make_shared>(); + std::vector m_table_connections; // This is shared between all clones of the output and must only be @@ -285,8 +295,7 @@ // accessed while protected using the lua_mutex. std::shared_ptr m_lua_state; - expire_config_t m_expire_config; - expire_tiles m_expire; + std::vector m_expire_tiles; way_cache_t m_way_cache; relation_cache_t m_relation_cache; diff -Nru osm2pgsql-1.8.1+ds/src/output-pgsql.cpp osm2pgsql-1.9.0+ds/src/output-pgsql.cpp --- osm2pgsql-1.8.1+ds/src/output-pgsql.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/output-pgsql.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -28,6 +28,7 @@ #include #include +#include "expire-output.hpp" #include "expire-tiles.hpp" #include "geom-from-osm.hpp" #include "geom-functions.hpp" @@ -146,18 +147,28 @@ } if (get_options()->expire_tiles_zoom_min > 0) { - auto const count = output_tiles_to_file( - m_expire.get_tiles(), get_options()->expire_tiles_zoom_min, - get_options()->expire_tiles_zoom, - get_options()->expire_tiles_filename); + expire_output_t expire_out; + expire_out.set_filename(get_options()->expire_tiles_filename); + expire_out.set_minzoom(get_options()->expire_tiles_zoom_min); + expire_out.set_maxzoom(get_options()->expire_tiles_zoom); + auto const count = + expire_out.output_tiles_to_file(m_expire.get_tiles()); log_info("Wrote {} entries to expired tiles list", count); } } void output_pgsql_t::wait() { + std::exception_ptr eptr; for (auto &t : m_tables) { - t->task_wait(); + try { + t->task_wait(); + } catch (...) { + eptr = std::current_exception(); + } + } + if (eptr) { + std::rethrow_exception(eptr); } } diff -Nru osm2pgsql-1.8.1+ds/src/overloaded.hpp osm2pgsql-1.9.0+ds/src/overloaded.hpp --- osm2pgsql-1.8.1+ds/src/overloaded.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/overloaded.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,25 @@ +#ifndef OSM2PGSQL_OVERLOADED_HPP +#define OSM2PGSQL_OVERLOADED_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +// This magic is used for visiting geometries. For an explanation see for +// instance here: +// https://arne-mertz.de/2018/05/overload-build-a-variant-visitor-on-the-fly/ +template +struct overloaded : Ts... +{ + using Ts::operator()...; +}; + +template +overloaded(Ts...) -> overloaded; + +#endif // OSM2PGSQL_OVERLOADED_HPP diff -Nru osm2pgsql-1.8.1+ds/src/pgsql-capabilities.cpp osm2pgsql-1.9.0+ds/src/pgsql-capabilities.cpp --- osm2pgsql-1.8.1+ds/src/pgsql-capabilities.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/pgsql-capabilities.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -32,6 +32,8 @@ char const *table, char const *column, char const *condition = "true") { + set->clear(); // Clear existing in case this is called multiple times + auto const res = db_connection.exec("SELECT {} FROM {} WHERE {}", column, table, condition); for (int i = 0; i < res.num_tuples(); ++i) { @@ -42,8 +44,10 @@ /// Get all config settings from the database. static void init_settings(pg_conn_t const &db_connection) { + capabilities().settings.clear(); // In case this is called multiple times + auto const res = - db_connection.exec("SELECT name, setting FROM pg_settings"); + db_connection.exec("SELECT name, setting FROM pg_catalog.pg_settings"); for (int i = 0; i < res.num_tuples(); ++i) { capabilities().settings.emplace(res.get(i, 0), res.get(i, 1)); @@ -66,7 +70,7 @@ { auto const res = db_connection.exec( "SELECT regexp_split_to_table(extversion, '\\.') FROM" - " pg_extension WHERE extname='postgis'"); + " pg_catalog.pg_extension WHERE extname='postgis'"); if (res.num_tuples() == 0) { throw fmt_error( @@ -122,6 +126,10 @@ "spcname != 'pg_global'"); init_set_from_query(&capabilities().index_methods, db_connection, "pg_catalog.pg_am", "amname", "amtype = 'i'"); + init_set_from_query( + &capabilities().tables, db_connection, "pg_catalog.pg_tables", + "schemaname || '.' || tablename", + "schemaname NOT IN ('pg_catalog', 'information_schema')"); } bool has_extension(std::string const &value) @@ -150,6 +158,25 @@ return capabilities().index_methods.count(value); } +bool has_table(std::string schema, std::string const &name) +{ + schema += '.'; + schema += name; + + return capabilities().tables.count(schema); +} + +void check_schema(std::string const &schema) +{ + if (has_schema(schema)) { + return; + } + + throw fmt_error("Schema '{0}' not available." + " Use 'CREATE SCHEMA \"{0}\";' to create it.", + schema); +} + uint32_t get_database_version() noexcept { return capabilities().database_version; diff -Nru osm2pgsql-1.8.1+ds/src/pgsql-capabilities.hpp osm2pgsql-1.9.0+ds/src/pgsql-capabilities.hpp --- osm2pgsql-1.8.1+ds/src/pgsql-capabilities.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/pgsql-capabilities.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,6 +10,7 @@ * For a full list of authors see the git log. */ +#include #include class pg_conn_t; @@ -20,6 +21,9 @@ bool has_schema(std::string const &value); bool has_tablespace(std::string const &value); bool has_index_method(std::string const &value); +bool has_table(std::string schema, std::string const &name); + +void check_schema(std::string const &schema); /// Get PostgreSQL version in the format (major * 10000 + minor). uint32_t get_database_version() noexcept; diff -Nru osm2pgsql-1.8.1+ds/src/pgsql-capabilities-int.hpp osm2pgsql-1.9.0+ds/src/pgsql-capabilities-int.hpp --- osm2pgsql-1.8.1+ds/src/pgsql-capabilities-int.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/pgsql-capabilities-int.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -24,6 +24,7 @@ std::set schemas; std::set tablespaces; std::set index_methods; + std::set tables; std::string database_name; diff -Nru osm2pgsql-1.8.1+ds/src/pgsql.cpp osm2pgsql-1.9.0+ds/src/pgsql.cpp --- osm2pgsql-1.8.1+ds/src/pgsql.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/pgsql.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -14,10 +14,18 @@ #include "util.hpp" #include +#include #include #include +#include #include +std::size_t pg_result_t::affected_rows() const noexcept +{ + char const *const s = PQcmdTuples(m_result.get()); + return std::strtoull(s, nullptr, 10); +} + std::atomic pg_conn_t::connection_id{0}; pg_conn_t::pg_conn_t(std::string const &conninfo) @@ -36,6 +44,12 @@ log_sql("(C{}) New database connection (backend_pid={})", m_connection_id, results.get(0, 0)); } + + // PostgreSQL sends notices in many different contexts which aren't that + // useful for the user. So we disable them for all connections. + if (!get_logger().debug_enabled()) { + exec("SET client_min_messages = WARNING"); + } } void pg_conn_t::close() @@ -56,8 +70,8 @@ // Update pg_settings instead of using SET because it does not yield // errors on older versions of PostgreSQL where the settings are not // implemented. - exec("UPDATE pg_settings SET setting = '{}' WHERE name = '{}'", value, - setting); + exec("UPDATE pg_catalog.pg_settings SET setting = '{}' WHERE name = '{}'", + value, setting); } pg_result_t pg_conn_t::exec(char const *sql) const @@ -169,7 +183,8 @@ if (status != PGRES_COMMAND_OK && status != PGRES_TUPLES_OK) { log_error("SQL command failed: EXECUTE {}({})", stmt, concat_params(num_params, param_values)); - throw fmt_error("Database error: {} ({})", error_msg(), status); + throw fmt_error("Database error: {} ({})", error_msg(), + std::underlying_type_t(status)); } return res; @@ -190,20 +205,8 @@ std::string qualified_name(std::string const &schema, std::string const &name) { - std::string result{"\""}; - - if (!schema.empty()) { - result.reserve(schema.size() + name.size() + 5); - result += schema; - result += "\".\""; - } else { - result.reserve(name.size() + 2); - } - - result += name; - result += '"'; - - return result; + assert(!schema.empty()); + return fmt::format(R"("{}"."{}")", schema, name); } void check_identifier(std::string const &name, char const *in) diff -Nru osm2pgsql-1.8.1+ds/src/pgsql-helper.cpp osm2pgsql-1.9.0+ds/src/pgsql-helper.cpp --- osm2pgsql-1.8.1+ds/src/pgsql-helper.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/pgsql-helper.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -25,13 +25,6 @@ return ids; } -idlist_t get_ids_from_db(pg_conn_t const *db_connection, char const *stmt, - osmid_t id) -{ - auto const res = db_connection->exec_prepared(stmt, id); - return get_ids_from_result(res); -} - void create_geom_check_trigger(pg_conn_t *db_connection, std::string const &schema, std::string const &table, @@ -80,15 +73,3 @@ auto const qual_name = qualified_name(schema, name); db_connection.exec("ANALYZE {}", qual_name); } - -bool has_table(pg_conn_t const &db_connection, std::string const &schema, - std::string const &table) -{ - auto const sql = fmt::format("SELECT count(*) FROM pg_tables" - " WHERE schemaname='{}' AND tablename='{}'", - schema.empty() ? "public" : schema, table); - auto const res = db_connection.exec(sql); - char const *const num = res.get_value(0, 0); - - return num[0] == '1' && num[1] == '\0'; -} diff -Nru osm2pgsql-1.8.1+ds/src/pgsql-helper.hpp osm2pgsql-1.9.0+ds/src/pgsql-helper.hpp --- osm2pgsql-1.8.1+ds/src/pgsql-helper.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/pgsql-helper.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -27,9 +27,6 @@ */ idlist_t get_ids_from_result(pg_result_t const &result); -idlist_t get_ids_from_db(pg_conn_t const *db_connection, char const *stmt, - osmid_t id); - void create_geom_check_trigger(pg_conn_t *db_connection, std::string const &schema, std::string const &table, @@ -42,11 +39,4 @@ void analyze_table(pg_conn_t const &db_connection, std::string const &schema, std::string const &name); -/** - * Check whether the table with the specified name exists in the specified - * schema in the database. Leave schema empty to check in the 'public' schema. - */ -bool has_table(pg_conn_t const &db_connection, std::string const &schema, - std::string const &table); - #endif // OSM2PGSQL_PGSQL_HELPER_HPP diff -Nru osm2pgsql-1.8.1+ds/src/pgsql.hpp osm2pgsql-1.9.0+ds/src/pgsql.hpp --- osm2pgsql-1.8.1+ds/src/pgsql.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/pgsql.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -13,8 +13,6 @@ /** * \file * - * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql). - * * Helper classes and functions for PostgreSQL access. */ @@ -25,9 +23,11 @@ #include #include #include +#include #include #include #include +#include #include /** @@ -66,6 +66,9 @@ return PQgetisnull(m_result.get(), row, col) != 0; } + /// Return the number of INSERTed, UPDATEd, or DELETEed rows. + std::size_t affected_rows() const noexcept; + /** * The length of the field at (row, col) in bytes. * @@ -124,6 +127,20 @@ }; /** + * Wrapper class for query parameters that should be sent to the database + * as binary parameter. + */ +class binary_param : public std::string_view +{ +public: + using std::string_view::string_view; + + explicit binary_param(std::string const &str) + : std::string_view(str.data(), str.size()) + {} +}; + +/** * PostgreSQL connection. * * Wraps the PGconn object of the libpq library. @@ -155,7 +172,7 @@ * status code PGRES_COMMAND_OK or PGRES_TUPLES_OK). */ template - pg_result_t exec(char const *sql, TArgs... params) const + pg_result_t exec(fmt::format_string sql, TArgs... params) const { return exec(fmt::format(sql, std::forward(params)...)); } @@ -222,9 +239,9 @@ template static constexpr std::size_t buffers_needed() noexcept { - if constexpr (std::is_same_v) { - return 0; - } else if constexpr (std::is_same_v) { + if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v) { return 0; } return 1; @@ -237,12 +254,18 @@ * strings. */ template - static char const *to_str(std::vector *data, T const ¶m) + static char const *to_str(std::vector *data, int *length, + int *bin, T const ¶m) { if constexpr (std::is_same_v) { return param; } else if constexpr (std::is_same_v) { + *length = param.size(); return param.c_str(); + } else if constexpr (std::is_same_v) { + *length = param.size(); + *bin = 1; + return param.data(); } return data->emplace_back(fmt::to_string(param)).c_str(); } @@ -272,16 +295,22 @@ std::vector exec_params; exec_params.reserve(total_buffers_needed); + std::array lengths = {0}; + std::array bins = {0}; + // This array holds the pointers to all parameter strings, either // to the original string parameters or to the recently converted // in the exec_params vector. + std::size_t n = 0; + std::size_t m = 0; std::array param_ptrs = { - to_str>(&exec_params, + to_str>(&exec_params, &lengths.at(n++), + &bins.at(m++), std::forward(params))...}; return exec_prepared_internal(stmt, sizeof...(params), - param_ptrs.data(), nullptr, nullptr, - result_as_binary ? 1 : 0); + param_ptrs.data(), lengths.data(), + bins.data(), result_as_binary ? 1 : 0); } struct pg_conn_deleter_t diff -Nru osm2pgsql-1.8.1+ds/src/progress-display.cpp osm2pgsql-1.9.0+ds/src/progress-display.cpp --- osm2pgsql-1.8.1+ds/src/progress-display.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/progress-display.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -22,7 +22,7 @@ return static_cast(count); } - return static_cast(count) / elapsed; + return static_cast(count) / static_cast(elapsed); } static std::string cps_display(std::size_t count, uint64_t elapsed) diff -Nru osm2pgsql-1.8.1+ds/src/progress-display.hpp osm2pgsql-1.9.0+ds/src/progress-display.hpp --- osm2pgsql-1.8.1+ds/src/progress-display.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/progress-display.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -13,12 +13,11 @@ /** * \file * - * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql). - * * It contains the progress_display_t class. */ #include +#include #include #include diff -Nru osm2pgsql-1.8.1+ds/src/properties.cpp osm2pgsql-1.9.0+ds/src/properties.cpp --- osm2pgsql-1.8.1+ds/src/properties.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/properties.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,167 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "properties.hpp" + +#include "format.hpp" +#include "logging.hpp" +#include "pgsql-capabilities.hpp" +#include "pgsql.hpp" + +#include +#include + +static constexpr char const *const properties_table = "osm2pgsql_properties"; + +properties_t::properties_t(std::string conninfo, std::string schema) +: m_conninfo(std::move(conninfo)), m_schema(std::move(schema)), + m_has_properties_table(has_table(m_schema, properties_table)) +{ + assert(!m_schema.empty()); + log_debug("Found properties table '{}': {}.", properties_table, + m_has_properties_table); +} + +std::string properties_t::get_string(std::string const &property, + std::string const &default_value) const +{ + auto const it = m_properties.find(property); + if (it == m_properties.end()) { + return default_value; + } + + return it->second; +} + +int64_t properties_t::get_int(std::string const &property, + int64_t default_value) const +{ + auto const it = m_properties.find(property); + if (it == m_properties.end()) { + return default_value; + } + + char *end = nullptr; + errno = 0; + int64_t const val = std::strtol(it->second.data(), &end, 10); + if (errno || *end != '\0') { + throw fmt_error("Corruption in properties: '{}' must be an integer.", + property); + } + + return val; +} + +bool properties_t::get_bool(std::string const &property, + bool default_value) const +{ + auto const it = m_properties.find(property); + if (it == m_properties.end()) { + return default_value; + } + + if (it->second == "true") { + return true; + } + if (it->second == "false") { + return false; + } + + throw fmt_error("Corruption in properties: '{}' must be 'true' or 'false'.", + property); +} + +void properties_t::set_string(std::string property, std::string value, + bool update_database) +{ + auto const r = + m_properties.insert_or_assign(std::move(property), std::move(value)); + + if (!update_database || !m_has_properties_table) { + return; + } + + auto const &inserted = *(r.first); + log_debug(" Storing {}='{}'", inserted.first, inserted.second); + + pg_conn_t const db_connection{m_conninfo}; + db_connection.exec( + "PREPARE set_property(text, text) AS" + " INSERT INTO {} (property, value) VALUES ($1, $2)" + " ON CONFLICT (property) DO UPDATE SET value = EXCLUDED.value", + table_name()); + db_connection.exec_prepared("set_property", inserted.first, + inserted.second); +} + +void properties_t::set_int(std::string property, int64_t value, + bool update_database) +{ + set_string(std::move(property), std::to_string(value), update_database); +} + +void properties_t::set_bool(std::string property, bool value, + bool update_database) +{ + set_string(std::move(property), value ? "true" : "false", update_database); +} + +void properties_t::store() +{ + auto const table = table_name(); + + log_info("Storing properties to table '{}'.", table); + pg_conn_t const db_connection{m_conninfo}; + + if (m_has_properties_table) { + db_connection.exec("TRUNCATE {}", table); + } else { + db_connection.exec("CREATE TABLE {} (" + " property TEXT NOT NULL PRIMARY KEY," + " value TEXT NOT NULL)", + table); + m_has_properties_table = true; + } + + db_connection.exec("PREPARE set_property(text, text) AS" + " INSERT INTO {} (property, value) VALUES ($1, $2)", + table); + + for (auto const &[k, v] : m_properties) { + log_debug(" Storing {}='{}'", k, v); + db_connection.exec_prepared("set_property", k, v); + } +} + +bool properties_t::load() +{ + if (!m_has_properties_table) { + log_info("No properties found in database from previous import."); + return false; + } + + m_properties.clear(); + + auto const table = table_name(); + log_info("Loading properties from table '{}'.", table); + + pg_conn_t const db_connection{m_conninfo}; + auto const result = db_connection.exec("SELECT * FROM {}", table); + + for (int i = 0; i < result.num_tuples(); ++i) { + m_properties.insert_or_assign(result.get_value(i, 0), result.get(i, 1)); + } + + return true; +} + +std::string properties_t::table_name() const +{ + return qualified_name(m_schema, properties_table); +} diff -Nru osm2pgsql-1.8.1+ds/src/properties.hpp osm2pgsql-1.9.0+ds/src/properties.hpp --- osm2pgsql-1.8.1+ds/src/properties.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/properties.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,104 @@ +#ifndef OSM2PGSQL_PROPERTIES_HPP +#define OSM2PGSQL_PROPERTIES_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * Osm2pgsql stores some properties in the database. Properties come from + * command line options or from the input files, they are used to keep the + * configuration consistent between imports and updates. + */ + +#include +#include +#include + +class pg_conn_t; + +class properties_t +{ +public: + /** + * Create new properties store. + * + * \param conninfo Connection info used to connect to the database. + * \param schema The schema used for storing the data, + * + * \pre You must have called init_database_capabilities() before this. + */ + properties_t(std::string conninfo, std::string schema); + + std::string get_string(std::string const &property, + std::string const &default_value) const; + + int64_t get_int(std::string const &property, int64_t default_value) const; + + bool get_bool(std::string const &property, bool default_value) const; + + /** + * Set property to string value. + * + * \param property Name of the property + * \param value Value of the property + * \param update_database Update database with this value immediately. + */ + void set_string(std::string property, std::string value, + bool update_database = false); + + /** + * Set property to integer value. The integer will be converted to a string + * internally. + * + * \param property Name of the property + * \param value Value of the property + * \param update_database Update database with this value immediately. + */ + void set_int(std::string property, int64_t value, + bool update_database = false); + + /** + * Set property to boolean value. In the database this will show up as the + * string 'true' or 'false'. + * + * \param property Name of the property + * \param value Value of the property + * \param update_database Update database with this value immediately. + */ + void set_bool(std::string property, bool value, + bool update_database = false); + + /** + * Store all properties in the database. Creates the properties table in + * the database if needed. Removes any properties that might already be + * stored in the database. + */ + void store(); + + /** + * Load all properties from the database. Clears any properties that might + * already exist before loading. + * + * \returns true if properties could be loaded, false otherwise. + */ + bool load(); + +private: + std::string table_name() const; + + std::map m_properties; + std::string m_conninfo; + std::string m_schema; + bool m_has_properties_table; + +}; // class properties_t + +#endif // OSM2PGSQL_PROPERTIES_HPP diff -Nru osm2pgsql-1.8.1+ds/src/reprojection.hpp osm2pgsql-1.9.0+ds/src/reprojection.hpp --- osm2pgsql-1.8.1+ds/src/reprojection.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/reprojection.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -2,11 +2,18 @@ #define OSM2PGSQL_REPROJECTION_HPP /** - * \file + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). * - * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql). + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file * - * It contains the reprojection class. + * Contains the reprojection class. */ #include "geom.hpp" diff -Nru osm2pgsql-1.8.1+ds/src/table.cpp osm2pgsql-1.9.0+ds/src/table.cpp --- osm2pgsql-1.8.1+ds/src/table.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/table.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -29,13 +30,11 @@ hstore_column hstore_mode, std::shared_ptr const ©_thread, std::string const &schema) -: m_target(std::make_shared(name.c_str(), "osm_id")), +: m_target(std::make_shared(schema, name, "osm_id")), m_type(std::move(type)), m_srid(fmt::to_string(srid)), m_append(append), m_hstore_mode(hstore_mode), m_columns(std::move(columns)), m_hstore_columns(std::move(hstore_columns)), m_copy(copy_thread) { - m_target->schema = schema; - // if we dont have any columns if (m_columns.empty() && m_hstore_mode != hstore_column::all) { throw fmt_error("No columns provided for table {}.", name); @@ -76,18 +75,17 @@ { if (m_sql_conn) { throw fmt_error("{} cannot start, its already started.", - m_target->name); + m_target->name()); } m_conninfo = conninfo; m_table_space = tablespace_clause(table_space); connect(); - log_info("Setting up table '{}'", m_target->name); - m_sql_conn->exec("SET client_min_messages = WARNING"); - auto const qual_name = qualified_name(m_target->schema, m_target->name); - auto const qual_tmp_name = qualified_name( - m_target->schema, m_target->name + "_tmp"); + log_info("Setting up table '{}'", m_target->name()); + auto const qual_name = qualified_name(m_target->schema(), m_target->name()); + auto const qual_tmp_name = + qualified_name(m_target->schema(), m_target->name() + "_tmp"); // we are making a new table if (!m_append) { @@ -96,7 +94,6 @@ // These _tmp tables can be left behind if we run out of disk space. m_sql_conn->exec("DROP TABLE IF EXISTS {}", qual_tmp_name); - m_sql_conn->exec("RESET client_min_messages"); //making a new table if (!m_append) { @@ -135,8 +132,8 @@ m_sql_conn->exec(sql); if (m_srid != "4326") { - create_geom_check_trigger(m_sql_conn.get(), m_target->schema, - m_target->name, "ST_IsValid(NEW.way)"); + create_geom_check_trigger(m_sql_conn.get(), m_target->schema(), + m_target->name(), "ST_IsValid(NEW.way)"); } } @@ -146,7 +143,7 @@ void table_t::prepare() { //let postgres cache this query as it will presumably happen a lot - auto const qual_name = qualified_name(m_target->schema, m_target->name); + auto const qual_name = qualified_name(m_target->schema(), m_target->name()); m_sql_conn->exec("PREPARE get_wkb(int8) AS" " SELECT way FROM {} WHERE osm_id = $1", qual_name); @@ -176,7 +173,7 @@ // add geom column joiner.add("way"); - m_target->rows = joiner(); + m_target->set_rows(joiner()); } void table_t::stop(bool updateable, bool enable_hstore_index, @@ -185,21 +182,17 @@ // make sure that all data is written to the DB before continuing m_copy.sync(); - auto const qual_name = qualified_name(m_target->schema, m_target->name); - auto const qual_tmp_name = qualified_name( - m_target->schema, m_target->name + "_tmp"); + auto const qual_name = qualified_name(m_target->schema(), m_target->name()); + auto const qual_tmp_name = + qualified_name(m_target->schema(), m_target->name() + "_tmp"); if (!m_append) { if (m_srid != "4326") { - drop_geom_check_trigger(m_sql_conn.get(), m_target->schema, - m_target->name); + drop_geom_check_trigger(m_sql_conn.get(), m_target->schema(), + m_target->name()); } - log_info("Clustering table '{}' by geometry...", m_target->name); - - // Notices about invalid geometries are expected and can be ignored - // because they say nothing about the validity of the geometry in OSM. - m_sql_conn->exec("SET client_min_messages = WARNING"); + log_info("Clustering table '{}' by geometry...", m_target->name()); std::string sql = fmt::format("CREATE TABLE {} {} AS SELECT * FROM {}", qual_tmp_name, m_table_space, qual_name); @@ -209,7 +202,7 @@ sql += " ORDER BY "; if (postgis_version.major == 2 && postgis_version.minor < 4) { log_debug("Using GeoHash for clustering table '{}'", - m_target->name); + m_target->name()); if (m_srid == "4326") { sql += "ST_GeoHash(way,10)"; } else { @@ -218,7 +211,7 @@ sql += " COLLATE \"C\""; } else { log_debug("Using native order for clustering table '{}'", - m_target->name); + m_target->name()); // Since Postgis 2.4 the order function for geometries gives // useful results. sql += "way"; @@ -228,9 +221,9 @@ m_sql_conn->exec("DROP TABLE {}", qual_name); m_sql_conn->exec(R"(ALTER TABLE {} RENAME TO "{}")", qual_tmp_name, - m_target->name); + m_target->name()); - log_info("Creating geometry index on table '{}'...", m_target->name); + log_info("Creating geometry index on table '{}'...", m_target->name()); // Use fillfactor 100 for un-updatable imports m_sql_conn->exec("CREATE INDEX ON {} USING GIST (way) {} {}", qual_name, @@ -239,12 +232,13 @@ /* slim mode needs this to be able to apply diffs */ if (updateable) { - log_info("Creating osm_id index on table '{}'...", m_target->name); + log_info("Creating osm_id index on table '{}'...", + m_target->name()); m_sql_conn->exec("CREATE INDEX ON {} USING BTREE (osm_id) {}", qual_name, tablespace_clause(table_space_index)); if (m_srid != "4326") { - create_geom_check_trigger(m_sql_conn.get(), m_target->schema, - m_target->name, + create_geom_check_trigger(m_sql_conn.get(), m_target->schema(), + m_target->name(), "ST_IsValid(NEW.way)"); } } @@ -252,7 +246,7 @@ /* Create hstore index if selected */ if (enable_hstore_index) { log_info("Creating hstore indexes on table '{}'...", - m_target->name); + m_target->name()); if (m_hstore_mode != hstore_column::none) { m_sql_conn->exec("CREATE INDEX ON {} USING GIN (tags) {}", qual_name, @@ -264,8 +258,8 @@ tablespace_clause(table_space_index)); } } - log_info("Analyzing table '{}'...", m_target->name); - analyze_table(*m_sql_conn, m_target->schema, m_target->name); + log_info("Analyzing table '{}'...", m_target->name()); + analyze_table(*m_sql_conn, m_target->schema(), m_target->name()); } teardown(); } @@ -376,7 +370,7 @@ void table_t::task_wait() { auto const run_time = m_task_result.wait(); - log_info("All postprocessing on table '{}' done in {}.", m_target->name, + log_info("All postprocessing on table '{}' done in {}.", m_target->name(), util::human_readable_duration(run_time)); } diff -Nru osm2pgsql-1.8.1+ds/src/tagtransform-c.cpp osm2pgsql-1.9.0+ds/src/tagtransform-c.cpp --- osm2pgsql-1.8.1+ds/src/tagtransform-c.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/tagtransform-c.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -53,7 +53,7 @@ int z_order = 0; - int const l = layer ? (int)strtol(layer->c_str(), nullptr, 10) : 0; + int const l = layer ? (int)std::strtol(layer->c_str(), nullptr, 10) : 0; z_order = 100 * l; *roads = false; diff -Nru osm2pgsql-1.8.1+ds/src/tagtransform.hpp osm2pgsql-1.9.0+ds/src/tagtransform.hpp --- osm2pgsql-1.8.1+ds/src/tagtransform.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/tagtransform.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -17,7 +17,7 @@ #include "osmtypes.hpp" class export_list; -class options_t; +struct options_t; class tagtransform_t { @@ -25,8 +25,15 @@ static std::unique_ptr make_tagtransform(options_t const *options, export_list const &exlist); + tagtransform_t() noexcept = default; + virtual ~tagtransform_t() = 0; + tagtransform_t(tagtransform_t const &) = delete; + tagtransform_t &operator=(tagtransform_t const &) = delete; + tagtransform_t(tagtransform_t &&) = delete; + tagtransform_t &operator=(tagtransform_t &&) = delete; + virtual std::unique_ptr clone() const = 0; virtual bool filter_tags(osmium::OSMObject const &o, bool *polygon, diff -Nru osm2pgsql-1.8.1+ds/src/tagtransform-lua.cpp osm2pgsql-1.9.0+ds/src/tagtransform-lua.cpp --- osm2pgsql-1.8.1+ds/src/tagtransform-lua.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/tagtransform-lua.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -7,12 +7,6 @@ * For a full list of authors see the git log. */ -extern "C" -{ -#include -#include -} - #include "format.hpp" #include "lua-utils.hpp" #include "tagtransform-lua.hpp" diff -Nru osm2pgsql-1.8.1+ds/src/tagtransform-lua.hpp osm2pgsql-1.9.0+ds/src/tagtransform-lua.hpp --- osm2pgsql-1.8.1+ds/src/tagtransform-lua.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/tagtransform-lua.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -14,10 +14,7 @@ #include "tagtransform.hpp" -extern "C" -{ -#include -} +#include class lua_tagtransform_t : public tagtransform_t { diff -Nru osm2pgsql-1.8.1+ds/src/thread-pool.cpp osm2pgsql-1.9.0+ds/src/thread-pool.cpp --- osm2pgsql-1.8.1+ds/src/thread-pool.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/thread-pool.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -9,8 +9,6 @@ #include "thread-pool.hpp" -#include - #include #include @@ -52,10 +50,7 @@ void thread_pool_t::worker_thread(unsigned int thread_num) { - std::string name{"_osm2pgsql_worker_"}; - name.append(std::to_string(thread_num)); - osmium::thread::set_thread_name(name.c_str()); - this_thread_num = thread_num + 1; + logger::init_thread(thread_num + 1); while (true) { osmium::thread::function_wrapper task; diff -Nru osm2pgsql-1.8.1+ds/src/thread-pool.hpp osm2pgsql-1.9.0+ds/src/thread-pool.hpp --- osm2pgsql-1.8.1+ds/src/thread-pool.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/thread-pool.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -13,8 +13,6 @@ /** * \file * - * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql). - * * Contains the class thread_pool_t. */ diff -Nru osm2pgsql-1.8.1+ds/src/tile.hpp osm2pgsql-1.9.0+ds/src/tile.hpp --- osm2pgsql-1.8.1+ds/src/tile.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/tile.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,8 +10,10 @@ * For a full list of authors see the git log. */ +#include "geom-box.hpp" #include "geom.hpp" +#include #include #include @@ -143,6 +145,56 @@ return half_earth_circumference - m_y * extent(); } + /// Same as box(margin).min_x(). + double xmin(double margin) const noexcept + { + return std::clamp(xmin() - margin * extent(), -half_earth_circumference, + half_earth_circumference); + } + + /// Same as box(margin).max_x(). + double xmax(double margin) const noexcept + { + return std::clamp(xmax() + margin * extent(), -half_earth_circumference, + half_earth_circumference); + } + + /// Same as box(margin).min_y(). + double ymin(double margin) const noexcept + { + return std::clamp(ymin() - margin * extent(), -half_earth_circumference, + half_earth_circumference); + } + + /// Same as box(margin).max_y(). + double ymax(double margin) const noexcept + { + return std::clamp(ymax() + margin * extent(), -half_earth_circumference, + half_earth_circumference); + } + + /** + * Get bounding box from tile. + */ + geom::box_t box() const noexcept + { + return {xmin(), ymin(), xmax(), ymax()}; + } + + /** + * Get bounding box from tile with margin on all sides. The margin is + * specified as a fraction of the tile extent. So margin==0.5 (50%) makes + * the bounding box twice as wide and twice as heigh. + * + * The bounding box is clamped to the extent of the earth, so there will + * be no coordinates smaller than -half_earth_circumference or larger than + * half_earth_circumference. + */ + geom::box_t box(double margin) const noexcept + { + return {xmin(margin), ymin(margin), xmax(margin), ymax(margin)}; + } + /** * Convert a point from web mercator (EPSG:3857) coordinates to * coordinates in the tile assuming a tile extent of `pixel_extent`. @@ -215,4 +267,55 @@ uint32_t m_zoom = invalid_zoom; }; // class tile_t +/** + * Iterate over tiles and call output function for each tile on all requested + * zoom levels. + * + * \tparam OUTPUT Class with operator() taking a tile_t argument + * + * \param tiles_at_maxzoom The list of tiles at maximum zoom level + * \param minzoom Minimum zoom level + * \param maxzoom Maximum zoom level + * \param output Output function + */ +template +std::size_t for_each_tile(quadkey_list_t const &tiles_at_maxzoom, + uint32_t minzoom, uint32_t maxzoom, OUTPUT &&output) +{ + assert(minzoom <= maxzoom); + + if (minzoom == maxzoom) { + for (auto const quadkey : tiles_at_maxzoom) { + std::forward(output)( + tile_t::from_quadkey(quadkey, maxzoom)); + } + return tiles_at_maxzoom.size(); + } + + /** + * Loop over all requested zoom levels (from maximum down to the minimum + * zoom level). + */ + quadkey_t last_quadkey{}; + std::size_t count = 0; + for (auto const quadkey : tiles_at_maxzoom) { + for (uint32_t dz = 0; dz <= maxzoom - minzoom; ++dz) { + auto const qt_current = quadkey.down(dz); + /** + * If dz > 0, there are probably multiple elements whose quadkey + * is equal because they are all sub-tiles of the same tile at the + * current zoom level. We skip all of them after we have written + * the first sibling. + */ + if (qt_current != last_quadkey.down(dz)) { + std::forward(output)( + tile_t::from_quadkey(qt_current, maxzoom - dz)); + ++count; + } + } + last_quadkey = quadkey; + } + return count; +} + #endif // OSM2PGSQL_TILE_HPP diff -Nru osm2pgsql-1.8.1+ds/src/util.cpp osm2pgsql-1.9.0+ds/src/util.cpp --- osm2pgsql-1.8.1+ds/src/util.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/util.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -23,11 +23,11 @@ std::string human_readable_duration(uint64_t seconds) { - if (seconds < 60) { + if (seconds < 60UL) { return fmt::format("{}s", seconds); } - if (seconds < (60 * 60)) { + if (seconds < (60UL * 60UL)) { return fmt::format("{}s ({}m {}s)", seconds, seconds / 60, seconds % 60); } diff -Nru osm2pgsql-1.8.1+ds/src/util.hpp osm2pgsql-1.9.0+ds/src/util.hpp --- osm2pgsql-1.8.1+ds/src/util.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/util.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -52,7 +52,11 @@ class timer_t { public: - timer_t() noexcept : m_start(clock::now()) {} + explicit timer_t(char const *name = "") + : m_name(name), m_start(clock::now()) + {} + + std::string const &name() const noexcept { return m_name; } void start() noexcept { m_start = clock::now(); } @@ -82,8 +86,19 @@ return static_cast(value) / static_cast(seconds); } + /** + * Add the elapsed time from the other timer to this one. This can be + * used, for instance, to aggregate timers used in different threads. + */ + void operator+=(timer_t const &other) noexcept + { + m_duration += other.m_duration; + } + private: using clock = std::chrono::steady_clock; + + std::string m_name; std::chrono::time_point m_start; std::chrono::microseconds m_duration{}; diff -Nru osm2pgsql-1.8.1+ds/src/version.cpp.in osm2pgsql-1.9.0+ds/src/version.cpp.in --- osm2pgsql-1.8.1+ds/src/version.cpp.in 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/version.cpp.in 2023-08-15 19:18:18.000000000 +0000 @@ -7,6 +7,8 @@ * For a full list of authors see the git log. */ +#include "version.hpp" + char const *get_build_type() noexcept { return "@CMAKE_BUILD_TYPE@"; @@ -27,7 +29,7 @@ return "@MINIMUM_POSTGRESQL_SERVER_VERSION@"; } -unsigned long get_minimum_postgresql_server_version_num() noexcept +uint32_t get_minimum_postgresql_server_version_num() noexcept { return @MINIMUM_POSTGRESQL_SERVER_VERSION_NUM@; } diff -Nru osm2pgsql-1.8.1+ds/src/version.hpp osm2pgsql-1.9.0+ds/src/version.hpp --- osm2pgsql-1.8.1+ds/src/version.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/version.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,10 +10,12 @@ * For a full list of authors see the git log. */ +#include + char const *get_build_type() noexcept; char const *get_osm2pgsql_version() noexcept; char const *get_osm2pgsql_short_version() noexcept; char const *get_minimum_postgresql_server_version() noexcept; -unsigned long get_minimum_postgresql_server_version_num() noexcept; +uint32_t get_minimum_postgresql_server_version_num() noexcept; #endif // OSM2PGSQL_VERSION_HPP diff -Nru osm2pgsql-1.8.1+ds/src/wkb.cpp osm2pgsql-1.9.0+ds/src/wkb.cpp --- osm2pgsql-1.8.1+ds/src/wkb.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/src/wkb.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -10,6 +10,7 @@ #include "wkb.hpp" #include "format.hpp" +#include "overloaded.hpp" #include #include @@ -231,13 +232,13 @@ if (m_ensure_multi) { // Two 13 bytes headers plus n sets of coordinates - data.reserve(2 * 13 + geom.size() * (2 * 8)); + data.reserve(2UL * 13UL + geom.size() * (2UL * 8UL)); write_header(&data, wkb_multi_line, m_srid); write_length(&data, 1); write_linestring(&data, geom); } else { // 13 byte header plus n sets of coordinates - data.reserve(13 + geom.size() * (2 * 8)); + data.reserve(13UL + geom.size() * (2UL * 8UL)); write_linestring(&data, geom, m_srid); } @@ -306,7 +307,7 @@ class ewkb_parser_t { public: - ewkb_parser_t(std::string_view input) + explicit ewkb_parser_t(std::string_view input) : m_data(input), m_max_length(static_cast(input.size()) / (sizeof(double) * 2)) {} diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/command-line/invalid.feature osm2pgsql-1.9.0+ds/tests/bdd/command-line/invalid.feature --- osm2pgsql-1.8.1+ds/tests/bdd/command-line/invalid.feature 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/command-line/invalid.feature 2023-08-15 19:18:18.000000000 +0000 @@ -14,3 +14,27 @@ --append and --create options can not be used at the same time """ + Scenario: append can only be used with slim mode + Then running osm2pgsql pgsql with parameters fails + | -a | + And the error output contains + """ + --append can only be used with slim mode + """ + + Scenario: append and middle-database-format cannot be used together + Then running osm2pgsql pgsql with parameters fails + | -a | --slim | --middle-database-format=new | + And the error output contains + """ + Do not use --middle-database-format with --append. + """ + + Scenario: middle-database-format value + Then running osm2pgsql pgsql with parameters fails + | --slim | --middle-database-format=foo | + And the error output contains + """ + Unknown value for --middle-database-format + """ + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/command-line/replication.feature osm2pgsql-1.9.0+ds/tests/bdd/command-line/replication.feature --- osm2pgsql-1.8.1+ds/tests/bdd/command-line/replication.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/command-line/replication.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,372 @@ +Feature: Tests for the osm2pgsql-replication script with property table + + Scenario: Replication can be initialised with an osm file after import + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 + """ + And the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | --osm-file={TEST_DATA_DIR}/liechtenstein-2013-08-03.osm.pbf | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 9999999 | + | replication_timestamp | 2013-08-03T19:00:02Z | + + + Scenario: Replication will be initialised from the information of the import file + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 9999999 | + | replication_timestamp | 2013-08-03T19:00:02Z | + + + Scenario: Replication cannot be initialised when date information is missing + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 + """ + When running osm2pgsql pgsql with parameters + | --slim | + + Then running osm2pgsql-replication fails with returncode 1 + | init | + And the error output contains + """ + Cannot get timestamp from database. + """ + + Scenario: Replication cannot initialised on non-updatable database + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + When running osm2pgsql pgsql + + Then running osm2pgsql-replication fails with returncode 1 + | init | + And the error output contains + """ + Database needs to be imported in --slim mode. + """ + + Scenario: Replication can be initialised for a database in a different schema (middle-schema) + Given the database schema foobar + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | --middle-schema=foobar | + + And running osm2pgsql-replication + | init | --middle-schema=foobar | + + Then table foobar.osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 9999999 | + | replication_timestamp | 2013-08-03T19:00:02Z | + + + Scenario: Replication can be initialised for a database in a different schema (schema) + Given the database schema foobar + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | --schema=foobar | + + And running osm2pgsql-replication + | init | --schema=foobar | + + Then table foobar.osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 9999999 | + | replication_timestamp | 2013-08-03T19:00:02Z | + + + Scenario: Replication can be initialised for a database in a different schema (schema) + Given the database schema foobar + Given the database schema baz + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | --middle-schema=foobar | --schema=baz | + + And running osm2pgsql-replication + | init | --middle-schema=foobar | --schema=baz | + + Then table foobar.osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 9999999 | + | replication_timestamp | 2013-08-03T19:00:02Z | + + + Scenario: Replication initialisation will fail for a database in a different schema + Given the database schema foobar + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | + + Then running osm2pgsql-replication fails with returncode 1 + | init | --middle-schema=foobar | + And the error output contains + """ + Database needs to be imported in --slim mode. + """ + + Scenario: Replication can be initialised with a fixed date (no previous replication info) + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 + """ + And the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-04T02:00:00Z | + | 347 | 2020-10-04T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | --start-at | 2020-10-04T01:30:00Z | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | https://planet.openstreetmap.org/replication/minute | + | replication_sequence_number | 345 | + | replication_timestamp | 2020-10-04T01:30:00Z | + + + Scenario: Replication can be initialised with a fixed date (with previous replication info) + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-04T02:00:00Z | + | 347 | 2020-10-04T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | --start-at | 2020-10-04T03:30:00Z | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 347 | + | replication_timestamp | 2020-10-04T03:30:00Z | + + + Scenario: Replication can be initialised from database date + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 t2020-10-04T04:00:01Z + """ + And the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-04T02:00:00Z | + | 347 | 2020-10-04T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | https://planet.openstreetmap.org/replication/minute | + | replication_sequence_number | 345 | + | replication_timestamp | 2020-10-04T01:00:01Z | + + + Scenario: Replication can be initialised with a rollback (no previous replication info) + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 t2020-10-04T02:00:01Z + """ + And the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-04T02:00:00Z | + | 347 | 2020-10-04T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | --start-at | 60 | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | https://planet.openstreetmap.org/replication/minute | + | replication_sequence_number | 345 | + | replication_timestamp | 2020-10-04T01:00:01Z | + + + Scenario: Replication can be initialised with a rollback (with previous replication info) + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-04T02:00:00Z | + | 347 | 2020-10-04T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | --start-at | 60 | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 345 | + | replication_timestamp | 2013-08-03T14:55:30Z | + + + Scenario: Replication can be initialised from a different server + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at https://custom.replication + | sequence | timestamp | + | 1345 | 2013-07-01T01:00:00Z | + | 1346 | 2013-08-01T01:00:00Z | + | 1347 | 2013-09-01T01:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | init | --server | https://custom.replication | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | https://custom.replication | + | replication_sequence_number | 1346 | + | replication_timestamp | 2013-08-03T12:55:30Z | + + + Scenario: Updates need an initialised replication + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 + """ + And the replication service at https://planet.openstreetmap.org/replication/minute + When running osm2pgsql pgsql with parameters + | --slim | + + Then running osm2pgsql-replication fails with returncode 1 + | update | + And the error output contains + """ + Updates not set up correctly. + """ + + Scenario: Updates run until the end (exactly one application) + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + | sequence | timestamp | + | 9999999 | 2013-08-01T01:00:02Z | + | 10000000 | 2013-09-01T01:00:00Z | + | 10000001 | 2013-10-01T01:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And running osm2pgsql-replication + | init | + And running osm2pgsql-replication + | update | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 10000001 | + | replication_timestamp | 2013-10-01T01:00:00Z | + + + Scenario: Updates run until the end (multiple applications) + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + | sequence | timestamp | + | 9999999 | 2013-08-01T01:00:02Z | + | 10000000 | 2013-09-01T01:00:00Z | + | 10000001 | 2013-10-01T01:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And running osm2pgsql-replication + | init | + And running osm2pgsql-replication + | update | --max-diff-size | 1 | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 10000001 | + | replication_timestamp | 2013-10-01T01:00:00Z | + + + Scenario: Updates can run only once + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + | sequence | timestamp | + | 9999999 | 2013-08-01T01:00:02Z | + | 10000000 | 2013-09-01T01:00:00Z | + | 10000001 | 2013-10-01T01:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And running osm2pgsql-replication + | init | + And running osm2pgsql-replication + | update | --once | --max-diff-size | 1 | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | http://example.com/europe/liechtenstein-updates | + | replication_sequence_number | 10000000 | + | replication_timestamp | 2013-09-01T01:00:00Z | + + + Scenario: Status of an uninitialised database fails + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 + """ + And the replication service at https://planet.openstreetmap.org/replication/minute + When running osm2pgsql pgsql with parameters + | --slim | + + Then running osm2pgsql-replication fails with returncode 2 + | status | --json | + And the standard output contains + """ + "status": 2 + "error": "Updates not set up correctly. + """ + + Scenario: Status of a freshly initialised database + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the replication service at http://example.com/europe/liechtenstein-updates + | sequence | timestamp | + | 9999999 | 2013-08-01T01:00:02Z | + | 10000000 | 2013-09-01T01:00:00Z | + | 10000001 | 2013-10-01T01:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + + And running osm2pgsql-replication + | status | --json | + Then the standard output contains + """ + "status": 0 + "server": {"base_url": "http://example.com/europe/liechtenstein-updates", "sequence": 10000001, "timestamp": "2013-10-01T01:00:00Z" + "local": {"sequence": 9999999, "timestamp": "2013-08-03T19:00:02Z" + """ diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/command-line/replication_legacy.feature osm2pgsql-1.9.0+ds/tests/bdd/command-line/replication_legacy.feature --- osm2pgsql-1.8.1+ds/tests/bdd/command-line/replication_legacy.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/command-line/replication_legacy.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,242 @@ +Feature: Tests for the osm2pgsql-replication script without property table + + Background: + Given the OSM data + """ + n34 Tamenity=restaurant x77 y45.3 + n35 x77 y45.31 + w4 Thighway=residential Nn34,n35 + """ + + Scenario: Replication can be initialised with a osm file + Given the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | + And deleting table osm2pgsql_properties + + And running osm2pgsql-replication + | init | --osm-file={TEST_DATA_DIR}/liechtenstein-2013-08-03.osm.pbf | + + Then table planet_osm_replication_status contains exactly + | url | sequence | importdate at time zone 'UTC' | + | http://example.com/europe/liechtenstein-updates | 9999999 | 2013-08-03 19:00:02 | + + + Scenario: Replication cannot be initialised from a osm file without replication info + Given the replication service at http://example.com/europe/liechtenstein-updates + When running osm2pgsql pgsql with parameters + | --slim | + And deleting table osm2pgsql_properties + + Then running osm2pgsql-replication fails with returncode 1 + | init | --osm-file={TEST_DATA_DIR}/008-ch.osc.gz | + And the error output contains + """ + has no usable replication headers + """ + + + Scenario: Replication can be initialised in different schema + Given the replication service at http://example.com/europe/liechtenstein-updates + Given the database schema foobar + When running osm2pgsql pgsql with parameters + | --slim | --middle-schema=foobar | + + And deleting table foobar.osm2pgsql_properties + + And running osm2pgsql-replication + | init | + | --osm-file={TEST_DATA_DIR}/liechtenstein-2013-08-03.osm.pbf | + | --middle-schema=foobar | + + Then table foobar.planet_osm_replication_status contains exactly + | url | sequence | importdate at time zone 'UTC' | + | http://example.com/europe/liechtenstein-updates | 9999999 | 2013-08-03 19:00:02 | + + + Scenario: Replication must be initialised in the same schema as rest of middle + Given the replication service at http://example.com/europe/liechtenstein-updates + Given the database schema foobar + When running osm2pgsql pgsql with parameters + | --slim | --middle-schema=foobar | + + And deleting table foobar.osm2pgsql_properties + + Then running osm2pgsql-replication fails with returncode 1 + | init | + | --osm-file={TEST_DATA_DIR}/liechtenstein-2013-08-03.osm.pbf | + And the error output contains + """ + Database needs to be imported in --slim mode. + """ + + Scenario: Replication can be initialised with a fixed start date + Given the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-14T02:00:00Z | + | 347 | 2020-10-24T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And deleting table osm2pgsql_properties + + And running osm2pgsql-replication + | init | --start-at | 2020-10-22T04:05:06Z | + + Then table planet_osm_replication_status contains exactly + | url | sequence | importdate at time zone 'UTC' | + | https://planet.openstreetmap.org/replication/minute | 346 | 2020-10-22 04:05:06 | + + + Scenario: Replication can be initialised from the data in the database + Given the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-14T02:00:00Z | + | 347 | 2020-10-24T03:00:00Z | + And the URL https://www.openstreetmap.org/api/0.6/way/4/1 returns + """ + {"version":"0.6","generator":"OpenStreetMap server","copyright":"OpenStreetMap and contributors","attribution":"http://www.openstreetmap.org/copyright","license":"http://opendatacommons.org/licenses/odbl/1-0/","elements":[{"type":"way","id":4,"timestamp":"2020-10-15T12:21:53Z","version":1,"changeset":234165,"user":"ewg2","uid":2,"nodes":[34,35],"tags":{"highway":"residential"}}]} + """ + When running osm2pgsql pgsql with parameters + | --slim | + And deleting table osm2pgsql_properties + + And running osm2pgsql-replication + | init | + + Then table planet_osm_replication_status contains exactly + | url | sequence | importdate at time zone 'UTC' | + | https://planet.openstreetmap.org/replication/minute | 346 | 2020-10-15 09:21:53 | + + + Scenario: Replication can be initialised from the data in the database with rollback + Given the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-14T02:00:00Z | + | 347 | 2020-10-24T03:00:00Z | + And the URL https://www.openstreetmap.org/api/0.6/way/4/1 returns + """ + {"version":"0.6","generator":"OpenStreetMap server","copyright":"OpenStreetMap and contributors","attribution":"http://www.openstreetmap.org/copyright","license":"http://opendatacommons.org/licenses/odbl/1-0/","elements":[{"type":"way","id":4,"timestamp":"2020-10-15T12:21:53Z","version":1,"changeset":234165,"user":"ewg2","uid":2,"nodes":[34,35],"tags":{"highway":"residential"}}]} + """ + When running osm2pgsql pgsql with parameters + | --slim | + And deleting table osm2pgsql_properties + + And running osm2pgsql-replication + | init | --start-at | 120 | + + Then table planet_osm_replication_status contains exactly + | url | sequence | importdate at time zone 'UTC' | + | https://planet.openstreetmap.org/replication/minute | 346 | 2020-10-15 10:21:53 | + + + Scenario: Updates need an initialised replication + Given the replication service at https://planet.openstreetmap.org/replication/minute + When running osm2pgsql pgsql with parameters + | --slim | + + Then running osm2pgsql-replication fails with returncode 1 + | update | + And the error output contains + """ + Updates not set up correctly. + """ + + Scenario: Updates run until the end (exactly one application) + Given the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-14T02:00:00Z | + | 347 | 2020-10-24T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And running osm2pgsql-replication + | init | --start-at | 2020-10-04T01:05:06Z | + And running osm2pgsql-replication + | update | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | https://planet.openstreetmap.org/replication/minute | + | replication_sequence_number | 347 | + | replication_timestamp | 2020-10-24T03:00:00Z | + + + Scenario: Updates run until the end (multiple applications) + Given the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-14T02:00:00Z | + | 347 | 2020-10-24T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And running osm2pgsql-replication + | init | --start-at | 2020-10-04T01:05:06Z | + And running osm2pgsql-replication + | update | --max-diff-size | 1 | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | https://planet.openstreetmap.org/replication/minute | + | replication_sequence_number | 347 | + | replication_timestamp | 2020-10-24T03:00:00Z | + + + Scenario: Updates can run only once + Given the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-14T02:00:00Z | + | 347 | 2020-10-24T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And running osm2pgsql-replication + | init | --start-at | 2020-10-04T01:05:06Z | + And running osm2pgsql-replication + | update | --max-diff-size | 1 | --once | + + Then table osm2pgsql_properties contains + | property | value | + | replication_base_url | https://planet.openstreetmap.org/replication/minute | + | replication_sequence_number | 346 | + | replication_timestamp | 2020-10-14T02:00:00Z | + + + + Scenario: Status of an uninitialised database fails + Given the replication service at https://planet.openstreetmap.org/replication/minute + When running osm2pgsql pgsql with parameters + | --slim | + And deleting table osm2pgsql_properties + + Then running osm2pgsql-replication fails with returncode 1 + | status | --json | + And the standard output contains + """ + "status": 1 + "error": "Cannot find replication status table + """ + + Scenario: Status of a freshly initialised database + Given the replication service at https://planet.openstreetmap.org/replication/minute + | sequence | timestamp | + | 345 | 2020-10-04T01:00:00Z | + | 346 | 2020-10-14T02:00:00Z | + | 347 | 2020-10-24T03:00:00Z | + When running osm2pgsql pgsql with parameters + | --slim | + And deleting table osm2pgsql_properties + + And running osm2pgsql-replication + | init | --start-at | 2020-10-22T04:05:06Z | + + And running osm2pgsql-replication + | status | --json | + Then the standard output contains + """ + "status": 0 + "server": {"base_url": "https://planet.openstreetmap.org/replication/minute", "sequence": 347, "timestamp": "2020-10-24T03:00:00Z" + "local": {"sequence": 346, "timestamp": "2020-10-22T04:05:06Z" + """ diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/environment.py osm2pgsql-1.9.0+ds/tests/bdd/environment.py --- osm2pgsql-1.8.1+ds/tests/bdd/environment.py 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/environment.py 2023-08-15 19:18:18.000000000 +0000 @@ -8,12 +8,16 @@ from pathlib import Path import subprocess import tempfile +import importlib.util +import io +from importlib.machinery import SourceFileLoader from behave import * import psycopg2 from psycopg2 import sql from steps.geometry_factory import GeometryFactory +from steps.replication_server_mock import ReplicationServerMock TEST_BASE_DIR = (Path(__file__) / '..' / '..').resolve() @@ -23,6 +27,7 @@ # behave -DBINARY=/tmp/my-builddir/osm2pgsql -DKEEP_TEST_DB USER_CONFIG = { 'BINARY': (TEST_BASE_DIR / '..' / 'build' / 'osm2pgsql').resolve(), + 'REPLICATION_SCRIPT': (TEST_BASE_DIR / '..' / 'scripts' / 'osm2pgsql-replication').resolve(), 'TEST_DATA_DIR': TEST_BASE_DIR / 'data', 'SRC_DIR': (TEST_BASE_DIR / '..').resolve(), 'KEEP_TEST_DB': False, @@ -88,6 +93,15 @@ context.test_data_dir = Path(context.config.userdata['TEST_DATA_DIR']).resolve() context.default_data_dir = Path(context.config.userdata['SRC_DIR']).resolve() + # Set up replication script. + replicationfile = str(Path(context.config.userdata['REPLICATION_SCRIPT']).resolve()) + spec = importlib.util.spec_from_loader('osm2pgsql_replication', + SourceFileLoader('osm2pgsql_replication', + replicationfile)) + assert spec, f"File not found: {replicationfile}" + context.osm2pgsql_replication = importlib.util.module_from_spec(spec) + spec.loader.exec_module(context.osm2pgsql_replication) + def before_scenario(context, scenario): """ Set up a fresh, empty test database. @@ -104,6 +118,16 @@ context.osm2pgsql_params = [] context.workdir = use_fixture(working_directory, context) context.geometry_factory = GeometryFactory() + context.osm2pgsql_replication.ReplicationServer = ReplicationServerMock() + context.urlrequest_responses = {} + + def _mock_urlopen(request): + if not request.full_url in context.urlrequest_responses: + raise urllib.error.URLError('Unknown URL') + + return closing(io.BytesIO(context.urlrequest_responses[request.full_url].encode('utf-8'))) + + context.osm2pgsql_replication.urlrequest.urlopen = _mock_urlopen @fixture @@ -129,6 +153,7 @@ with tempfile.TemporaryDirectory() as tmpdir: yield Path(tmpdir) + def before_tag(context, tag): if tag == 'needs-pg-index-includes': if context.config.userdata['PG_VERSION'] < 110000: diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/flex/extra-attributes.feature osm2pgsql-1.9.0+ds/tests/bdd/flex/extra-attributes.feature --- osm2pgsql-1.8.1+ds/tests/bdd/flex/extra-attributes.feature 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/flex/extra-attributes.feature 2023-08-15 19:18:18.000000000 +0000 @@ -1,14 +1,7 @@ Feature: Tests for including extra attributes Background: - Given the grid - | 11 | 12 | - | 10 | | - And the OSM data - """ - w20 v1 dV c31 t2020-01-12T12:34:56Z i17 utest Thighway=primary Nn10,n11,n12 - """ - And the lua style + Given the lua style """ local attr_table = osm2pgsql.define_table{ name = 'osm2pgsql_test_attr', @@ -32,12 +25,19 @@ """ Scenario: Importing data without extra attributes + Given the grid + | 11 | 12 | + | 10 | | + And the OSM data + """ + w20 v1 dV c31 t2020-01-12T12:34:56Z i17 utest Thighway=primary Nn10,n11,n12 + """ When running osm2pgsql flex with parameters | --slim | Then table osm2pgsql_test_attr contains - | type | way_id | tags->'highway' | version | changeset | timestamp | uid | "user" | - | way | 20 | primary | NULL | NULL | NULL | NULL | NULL | + | type | way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp | uid | "user" | + | way | 20 | primary | NULL | NULL | NULL | NULL | NULL | NULL | Given the grid | | | @@ -46,17 +46,24 @@ | --slim | --append | Then table osm2pgsql_test_attr contains - | type | way_id | tags->'highway' | version | changeset | timestamp | uid | "user" | - | way | 20 | primary | NULL | NULL | NULL | NULL | NULL | + | type | way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp | uid | "user" | + | way | 20 | primary | NULL | NULL | NULL | NULL | NULL | NULL | Scenario: Importing data with extra attributes + Given the grid + | 11 | 12 | + | 10 | | + And the OSM data + """ + w20 v1 dV c31 t2020-01-12T12:34:56Z i17 utest Thighway=primary Nn10,n11,n12 + """ When running osm2pgsql flex with parameters | --slim | -x | Then table osm2pgsql_test_attr contains - | type | way_id | tags->'highway' | version | changeset | timestamp | uid | "user" | - | way | 20 | primary | 1 | 31 | 1578832496 | 17 | test | + | type | way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp | uid | "user" | + | way | 20 | primary | NULL | 1 | 31 | 1578832496 | 17 | test | Given the grid | | | @@ -65,6 +72,6 @@ | --slim | --append | -x | Then table osm2pgsql_test_attr contains - | type | way_id | tags->'highway' | version | changeset | timestamp | uid | "user" | - | way | 20 | primary | 1 | 31 | 1578832496 | 17 | test | + | type | way_id | tags->'highway' | tags->'osm_version' | version | changeset | timestamp | uid | "user" | + | way | 20 | primary | NULL | 1 | 31 | 1578832496 | 17 | test | diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/flex/lua-expire.feature osm2pgsql-1.9.0+ds/tests/bdd/flex/lua-expire.feature --- osm2pgsql-1.8.1+ds/tests/bdd/flex/lua-expire.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/flex/lua-expire.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,240 @@ +Feature: Expire configuration in Lua file + + Scenario: Expire on a table must be on geometry column + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + osm2pgsql.define_node_table('bar', { + { column = 'some', expire = {{ output = eo }} } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Expire only allowed for geometry columns in Web Mercator projection. + """ + + Scenario: Expire on a table must be on geometry column in 3857 + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + projection = 4326, + expire = {{ output = eo }} } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Expire only allowed for geometry columns in Web Mercator projection. + """ + + Scenario: Expire reference must be a userdata ExpireOutput object + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + osm2pgsql.define_node_table('bar', { + { column = 'some', type = 'geometry', expire = {{ output = 'abc' }} } + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Expire output must be of type ExpireOutput. + """ + + Scenario: Expire on a table with 3857 geometry is okay + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + local t = osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = {{ output = eo }} } + }) + + function osm2pgsql.process_node(object) + t:insert({}) + end + """ + When running osm2pgsql flex + Then table bar has 1562 rows + + Scenario: Directly specifying the expire output is okay + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + local t = osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = eo } + }) + + function osm2pgsql.process_node(object) + t:insert({}) + end + """ + When running osm2pgsql flex + Then table bar has 1562 rows + + Scenario: Expire with buffer option that's not a number fails + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = { + { output = eo, buffer = 'notvalid' } + }} + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Optional expire field 'buffer' must contain a number. + """ + + Scenario: Expire with invalid mode setting fails + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + local t = osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = {{ output = eo, mode = 'foo' }} } + }) + + function osm2pgsql.process_node(object) + t:insert({}) + end + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Unknown expire mode 'foo'. + """ + + Scenario: Expire with full_area_limit that's not a number fails + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = { + { output = eo, full_area_limit = true } + }} + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Optional expire field 'full_area_limit' must contain a number. + """ + + Scenario: Expire with boundary-only options is okay + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + local t = osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = { + { output = eo, buffer = 0.2, mode = 'boundary-only' } + }} + }) + + function osm2pgsql.process_node(object) + t:insert({}) + end + """ + When running osm2pgsql flex + Then table bar has 1562 rows + + Scenario: Expire with hybrid options is okay + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + filename = 'bar', + maxzoom = 12 + }) + local t = osm2pgsql.define_node_table('bar', { + { column = 'some', + type = 'geometry', + expire = { + { output = eo, buffer = 0.2, + mode = 'hybrid', full_area_limit = 10000 } + }} + }) + + function osm2pgsql.process_node(object) + t:insert({}) + end + """ + When running osm2pgsql flex + Then table bar has 1562 rows + + Scenario: Expire into table is okay + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + local eo = osm2pgsql.define_expire_output({ + table = 'tiles', + maxzoom = 12 + }) + local t = osm2pgsql.define_node_table('nodes', { + { column = 'geom', + type = 'point', + expire = { + { output = eo } + }} + }) + + function osm2pgsql.process_node(object) + t:insert({}) + end + """ + When running osm2pgsql flex + Then table nodes has 1562 rows + And table tiles has 0 rows + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/flex/lua-expire-output-definitions.feature osm2pgsql-1.9.0+ds/tests/bdd/flex/lua-expire-output-definitions.feature --- osm2pgsql-1.8.1+ds/tests/bdd/flex/lua-expire-output-definitions.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/flex/lua-expire-output-definitions.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,143 @@ +Feature: Expire output definitions in Lua file + + Scenario: Expire output definition needs a Lua table parameter + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output() + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Argument #1 to 'define_expire_output' must be a Lua table. + """ + + Scenario: Filename in expire output definition has to be a string + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + filename = false + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + The expire output field must contain a 'filename' string field (or nil for default: ''). + """ + + Scenario: Schema in expire output definition has to be a string + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + table = 'bar', + schema = false + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + The expire output field must contain a 'schema' string field (or nil for default: 'public'). + """ + + Scenario: Table in expire output definition has to be a string + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + table = false + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + The expire output field must contain a 'table' string field (or nil for default: ''). + """ + + Scenario: Maxzoom value in expire output definition has to be an integer + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + maxzoom = 'bar', + filename = 'somewhere' + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + The 'maxzoom' field in a expire output must contain an integer. + """ + + Scenario: Minzoom value in expire output definition has to be an integer + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + maxzoom = 12, + minzoom = 'bar', + filename = 'somewhere' + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + The 'minzoom' field in a expire output must contain an integer. + """ + + Scenario: Maxzoom value in expire output definition has to be in range + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + maxzoom = 123, + filename = 'somewhere' + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Value of 'maxzoom' field must be between 1 and 20. + """ + + Scenario: Minzoom value in expire output definition has to be in range + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + maxzoom = 12, + minzoom = -3, + filename = 'somewhere' + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Value of 'minzoom' field must be between 1 and 'maxzoom'. + """ + + Scenario: Minzoom value in expire output definition has to be smaller than maxzoom + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + And the lua style + """ + osm2pgsql.define_expire_output({ + name = 'foo', + maxzoom = 12, + minzoom = 14, + filename = 'somewhere' + }) + """ + Then running osm2pgsql flex fails + And the error output contains + """ + Value of 'minzoom' field must be between 1 and 'maxzoom'. + """ + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/flex/lua-index-definitions.feature osm2pgsql-1.9.0+ds/tests/bdd/flex/lua-index-definitions.feature --- osm2pgsql-1.8.1+ds/tests/bdd/flex/lua-index-definitions.feature 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/flex/lua-index-definitions.feature 2023-08-15 19:18:18.000000000 +0000 @@ -362,7 +362,7 @@ Then running osm2pgsql flex fails And the error output contains """ - Index definition field 'tablespace' must be a string field. + Index definition field must contain a 'tablespace' string field (or nil for default: ''). """ Scenario: Empty tablespace is okay @@ -452,7 +452,7 @@ Then running osm2pgsql flex fails And the error output contains """ - Index definition field 'where' must be a string field. + Index definition field must contain a 'where' string field (or nil for default: ''). """ Scenario: Where condition works diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/flex/run-with-expire.feature osm2pgsql-1.9.0+ds/tests/bdd/flex/run-with-expire.feature --- osm2pgsql-1.8.1+ds/tests/bdd/flex/run-with-expire.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/flex/run-with-expire.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,41 @@ +Feature: Expire into table + + Background: + Given the lua style + """ + local eo = osm2pgsql.define_expire_output({ + name = 'tiles', + table = 'tiles', + maxzoom = 12 + }) + local t = osm2pgsql.define_node_table('nodes', { + { column = 'geom', type = 'point', expire = eo }, + { column = 'tags', type = 'jsonb' } + }) + + function osm2pgsql.process_node(object) + if object.tags then + t:insert({ + tags = object.tags, + geom = object:as_point() + }) + end + end + """ + + Scenario: Expire into table in append mode + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + When running osm2pgsql flex with parameters + | --slim | -c | + Then table nodes has 1562 rows + And table tiles has 0 rows + + Given the OSM data + """ + n27 v1 dV x10 y10 Tamenity=restaurant + """ + When running osm2pgsql flex with parameters + | --slim | -a | + Then table nodes has 1563 rows + And table tiles has 1 rows + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/regression/extra-attributes.feature osm2pgsql-1.9.0+ds/tests/bdd/regression/extra-attributes.feature --- osm2pgsql-1.8.1+ds/tests/bdd/regression/extra-attributes.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/regression/extra-attributes.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,53 @@ +Feature: Tests for including extra attributes + + Scenario: Importing data without extra attributes + Given the grid + | 11 | 12 | + | 10 | | + And the OSM data + """ + w20 v1 dV c31 t2020-01-12T12:34:56Z i17 utest Thighway=primary Nn10,n11,n12 + """ + When running osm2pgsql pgsql with parameters + | --slim | -j | + + Then table planet_osm_roads contains + | osm_id | tags->'highway' | tags->'osm_version' | tags->'osm_changeset' | + | 20 | primary | NULL | NULL | + + Given the grid + | | | + | | 10 | + When running osm2pgsql pgsql with parameters + | --slim | -j | --append | + + Then table planet_osm_roads contains + | osm_id | tags->'highway' | tags->'osm_version' | tags->'osm_changeset' | + | 20 | primary | NULL | NULL | + + + Scenario: Importing data with extra attributes + Given the grid + | 11 | 12 | + | 10 | | + And the OSM data + """ + w20 v1 dV c31 t2020-01-12T12:34:56Z i17 utest Thighway=primary Nn10,n11,n12 + """ + When running osm2pgsql pgsql with parameters + | --slim | -j | -x | + + Then table planet_osm_roads contains + | osm_id | tags->'highway' | tags->'osm_version' | tags->'osm_changeset' | + | 20 | primary | 1 | 31 | + + Given the grid + | | | + | | 10 | + When running osm2pgsql pgsql with parameters + | --slim | -j | --append | -x | + + Then table planet_osm_roads contains + | osm_id | tags->'highway' | tags->'osm_version' | tags->'osm_changeset' | + | 20 | primary | 1 | 31 | + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/regression/import.feature osm2pgsql-1.9.0+ds/tests/bdd/regression/import.feature --- osm2pgsql-1.8.1+ds/tests/bdd/regression/import.feature 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/regression/import.feature 2023-08-15 19:18:18.000000000 +0000 @@ -149,9 +149,12 @@ | --tablespace-slim-index| tablespacetest | | - Scenario: Import slim with hstore and extra tags + Scenario Outline: Import slim with hstore and extra tags When running osm2pgsql pgsql with parameters - | --slim | -j | -x | + | --slim | + | -j | + | -x | + | | Then table planet_osm_point has 1360 rows with condition """ @@ -182,3 +185,8 @@ tags ? 'osm_changeset' """ + Examples: Middle database format + | middle_format | + | --middle-database-format=legacy | + | --middle-database-format=new | + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/regression/multipolygon.feature osm2pgsql-1.9.0+ds/tests/bdd/regression/multipolygon.feature --- osm2pgsql-1.8.1+ds/tests/bdd/regression/multipolygon.feature 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/regression/multipolygon.feature 2023-08-15 19:18:18.000000000 +0000 @@ -141,8 +141,6 @@ When running osm2pgsql pgsql with parameters | --slim | -F | flat.store | - Then there is no table planet_osm_nodes - Then table planet_osm_polygon contains | osm_id | landuse | name | round(ST_Area(way)) | | -1 | residential | Name_rel | 12895 | diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/regression/properties.feature osm2pgsql-1.9.0+ds/tests/bdd/regression/properties.feature --- osm2pgsql-1.8.1+ds/tests/bdd/regression/properties.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/regression/properties.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,124 @@ +Feature: Updates to the test database with properties check + + Background: + Given the input file 'liechtenstein-2013-08-03.osm.pbf' + + Scenario Outline: Create/append with various parameters + When running osm2pgsql pgsql with parameters + | -c | + | | + + Given the input file '000466354.osc.gz' + Then running osm2pgsql pgsql with parameters fails + | -a | + | --slim | + | | + And the error output contains + """ + + """ + + Examples: + | param_create | param_append | message | + | | | This database is not updatable | + | --slim | -x | because original import was without attributes | + | --slim | --prefix=foo | Different prefix specified | + | --slim | --flat-nodes=x | Database was imported without flat node file | + + + Scenario: Append without output on null output + When running osm2pgsql null with parameters + | -c | + | --slim | + + Given the input file '000466354.osc.gz' + When running osm2pgsql nooutput with parameters + | -a | + | --slim | + Then the error output contains + """ + Using output 'null' (same as on import). + """ + + + Scenario Outline: Create/append with various parameters + When running osm2pgsql pgsql with parameters + | --slim | + | | + + Given the input file '000466354.osc.gz' + When running osm2pgsql pgsql with parameters + | -a | + | --slim | + | | + Then the error output contains + """ + + """ + + Examples: + | param_create | param_append | message | + | -x | | Updating with attributes (same as on import). | + | | | Not using flat node file (same as on import). | + | --flat-nodes=x | | Using flat node file | + | --flat-nodes=x | --flat-nodes=x | Using flat node file | + | --flat-nodes=x | --flat-nodes=y | Using the flat node file you specified | + | --prefix=abc | | Using prefix 'abc' (same as on import). | + + + Scenario: Create with different output than append + When running osm2pgsql pgsql with parameters + | --slim | + + Given the input file '000466354.osc.gz' + Then running osm2pgsql null with parameters fails + | -a | + | --slim | + And the error output contains + """ + Different output specified on command line + """ + + Scenario Outline: Create/append with with null output doesn't need style + When running osm2pgsql null with parameters + | --slim | + + Given the input file '000466354.osc.gz' + When running osm2pgsql null with parameters + | -a | + | --slim | + | | + Then the error output contains + """ + + """ + + Examples: + | param | message | + | | Using style file '' (same as on import). | + | --style= | Using style file '' (same as on import). | + + + @config.have_lua + Scenario Outline: Create/append with various style parameters with flex output + When running osm2pgsql flex with parameters + | --slim | + | | + + Given the input file '000466354.osc.gz' + When running osm2pgsql flex with parameters + | -a | + | --slim | + | | + Then the error output contains + """ + + """ + + Examples: + | param_create | param_append | message | + | --style={TEST_DATA_DIR}/test_output_flex.lua | | Using style file | + | --style={TEST_DATA_DIR}/test_output_flex.lua | --style={TEST_DATA_DIR}/test_output_flex.lua | Using style file | + | --style={TEST_DATA_DIR}/test_output_flex.lua | --style={TEST_DATA_DIR}/test_output_flex_copy.lua | Using the style file you specified | + + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/regression/timestamps.feature osm2pgsql-1.9.0+ds/tests/bdd/regression/timestamps.feature --- osm2pgsql-1.8.1+ds/tests/bdd/regression/timestamps.feature 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/regression/timestamps.feature 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,171 @@ +Feature: Timestamps in properties table should reflect timestamps in input file + + Scenario: Create database with timestamps + Given the OSM data + """ + n10 t2020-01-02T03:04:05Z x10 y10 + n11 t2020-01-02T03:04:05Z x10 y11 + w20 t2020-01-02T03:04:06Z Thighway=primary Nn10,n11 + """ + When running osm2pgsql pgsql + + Then table osm2pgsql_properties has 10 rows + And table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 0 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | false | + | import_timestamp | 2020-01-02T03:04:06Z | + | current_timestamp | 2020-01-02T03:04:06Z | + | output | pgsql | + + Scenario: Create database without timestamps + Given the OSM data + """ + n10 x10 y10 + n11 x10 y11 + w20 Thighway=primary Nn10,n11 + """ + When running osm2pgsql pgsql + + Then table osm2pgsql_properties has 8 rows + Then table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 0 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | false | + | output | pgsql | + + Scenario: Create and update database with timestamps + Given the OSM data + """ + n10 v1 t2020-01-02T03:04:05Z x10 y10 + n11 v1 t2020-01-02T03:04:05Z x10 y11 + w20 v1 t2020-01-02T03:04:06Z Thighway=primary Nn10,n11 + """ + + When running osm2pgsql pgsql with parameters + | --create | --slim | + + Then table osm2pgsql_properties has 10 rows + And table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 1 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | true | + | import_timestamp | 2020-01-02T03:04:06Z | + | current_timestamp | 2020-01-02T03:04:06Z | + | output | pgsql | + + Given the OSM data + """ + n11 v2 t2020-01-02T03:06:05Z x10 y12 + w20 v2 t2020-01-02T03:05:06Z Thighway=secondary Nn10,n11 + """ + + When running osm2pgsql pgsql with parameters + | --append | --slim | + + Then table osm2pgsql_properties has 10 rows + And table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 1 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | true | + | import_timestamp | 2020-01-02T03:04:06Z | + | current_timestamp | 2020-01-02T03:06:05Z | + | output | pgsql | + + Scenario: Create database with timestamps and update without timestamps + Given the OSM data + """ + n10 v1 t2020-01-02T03:04:05Z x10 y10 + n11 v1 t2020-01-02T03:04:05Z x10 y11 + w20 v1 t2020-01-02T03:04:06Z Thighway=primary Nn10,n11 + """ + + When running osm2pgsql pgsql with parameters + | --create | --slim | + + Then table osm2pgsql_properties has 10 rows + And table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 1 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | true | + | import_timestamp | 2020-01-02T03:04:06Z | + | current_timestamp | 2020-01-02T03:04:06Z | + | output | pgsql | + + Given the OSM data + """ + n11 v2 x10 y12 + w20 v2 Thighway=secondary Nn10,n11 + """ + + When running osm2pgsql pgsql with parameters + | --append | --slim | + + Then table osm2pgsql_properties has 10 rows + And table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 1 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | true | + | import_timestamp | 2020-01-02T03:04:06Z | + | current_timestamp | 2020-01-02T03:04:06Z | + | output | pgsql | + + Scenario: Create database without timestamps and update with timestamps + Given the OSM data + """ + n10 v1 x10 y10 + n11 v1 x10 y11 + w20 v1 Thighway=primary Nn10,n11 + """ + + When running osm2pgsql pgsql with parameters + | --create | --slim | + + Then table osm2pgsql_properties has 8 rows + And table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 1 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | true | + | output | pgsql | + + Given the OSM data + """ + n11 v2 t2020-01-02T03:06:05Z x10 y12 + w20 v2 t2020-01-02T03:05:06Z Thighway=secondary Nn10,n11 + """ + + When running osm2pgsql pgsql with parameters + | --append | --slim | + + Then table osm2pgsql_properties has 9 rows + And table osm2pgsql_properties contains + | property | value | + | attributes | false | + | db_format | 1 | + | flat_node_file | | + | prefix | planet_osm | + | updatable | true | + | current_timestamp | 2020-01-02T03:06:05Z | + | output | pgsql | + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/regression/update.feature osm2pgsql-1.9.0+ds/tests/bdd/regression/update.feature --- osm2pgsql-1.8.1+ds/tests/bdd/regression/update.feature 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/regression/update.feature 2023-08-15 19:18:18.000000000 +0000 @@ -22,6 +22,7 @@ And table planet_osm_line has 3274 rows And table planet_osm_roads has 380 rows And table planet_osm_polygon has 4277 rows + And table osm2pgsql_properties has 13 rows Examples: | param1 | param2 | param3 | diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/steps/replication_server_mock.py osm2pgsql-1.9.0+ds/tests/bdd/steps/replication_server_mock.py --- osm2pgsql-1.8.1+ds/tests/bdd/steps/replication_server_mock.py 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/steps/replication_server_mock.py 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# This file is part of osm2pgsql (https://osm2pgsql.org/). +# +# Copyright (C) 2023 by the osm2pgsql developer community. +# For a full list of authors see the git log. + + +class ReplicationServerMock: + + def __init__(self): + self.expected_base_url = None + self.state_infos = [] + + + def __call__(self, base_url): + assert self.expected_base_url is not None and base_url == self.expected_base_url,\ + f"Wrong replication service called. Expected '{self.expected_base_url}', got '{base_url}'" + return self + + + def get_state_info(self, seq=None, retries=2): + assert self.state_infos, 'Replication mock not properly set up' + if seq is None: + return self.state_infos[-1] + + for info in self.state_infos: + if info.sequence == seq: + return info + + assert False, f"No sequence information for sequence ID {seq}." + + def timestamp_to_sequence(self, timestamp, balanced_search=False): + assert self.state_infos, 'Replication mock not properly set up' + + if timestamp < self.state_infos[0].timestamp: + return self.state_infos[0].sequence + + prev = self.state_infos[0] + for info in self.state_infos: + if timestamp >= prev.timestamp and timestamp < info.timestamp: + return prev.sequence + prev = info + + return prev.sequence + + def apply_diffs(self, handler, start_id, max_size=1024, idx="", simplify=True): + if start_id > self.state_infos[-1].sequence: + return None + + numdiffs = int((max_size + 1023)/1024) + return min(self.state_infos[-1].sequence, start_id + numdiffs - 1) + diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/steps/steps_db.py osm2pgsql-1.9.0+ds/tests/bdd/steps/steps_db.py --- osm2pgsql-1.8.1+ds/tests/bdd/steps/steps_db.py 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/steps/steps_db.py 2023-08-15 19:18:18.000000000 +0000 @@ -18,16 +18,18 @@ with context.db.cursor() as cur: cur.execute("CREATE SCHEMA " + schema) + +@when("deleting table (?P.+)") +def delete_table(context, table): + with context.db.cursor() as cur: + cur.execute("DROP TABLE " + table) + + @then("table (?P
.+) has (?P\d+) rows?(?P with condition)?") def db_table_row_count(context, table, row_num, has_where): assert table_exists(context.db, table) - if '.' in table: - schema, tablename = table.split('.', 2) - query = sql.SQL("SELECT count(*) FROM {}.{}")\ - .format(sql.Identifier(schema), sql.Identifier(tablename)) - else: - query = sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(table)) + query = sql.SQL("SELECT count(*) FROM {}").format(sql.Identifier(*table.split('.', 2))) if has_where: query = sql.SQL("{} WHERE {}").format(query, sql.SQL(context.text)) @@ -43,7 +45,7 @@ assert table_exists(context.db, table) query = sql.SQL("SELECT round(sum({})) FROM {}")\ - .format(sql.SQL(formula), sql.Identifier(table)) + .format(sql.SQL(formula), sql.Identifier(*table.split('.', 2))) if has_where: query = sql.SQL("{} WHERE {}").format(query, sql.SQL(context.text)) @@ -72,7 +74,8 @@ rows = sql.SQL(', '.join(h.rsplit('@')[0] for h in context.table.headings)) with context.db.cursor() as cur: - cur.execute(sql.SQL("SELECT {} FROM {}").format(rows, sql.Identifier(table))) + cur.execute(sql.SQL("SELECT {} FROM {}") + .format(rows, sql.Identifier(*table.split('.', 2)))) actuals = list(DBRow(r, context.table.headings, context.geometry_factory) for r in cur) diff -Nru osm2pgsql-1.8.1+ds/tests/bdd/steps/steps_execute.py osm2pgsql-1.9.0+ds/tests/bdd/steps/steps_execute.py --- osm2pgsql-1.8.1+ds/tests/bdd/steps/steps_execute.py 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/bdd/steps/steps_execute.py 2023-08-15 19:18:18.000000000 +0000 @@ -9,7 +9,13 @@ """ from io import StringIO from pathlib import Path +import sys import subprocess +import contextlib +import logging +import datetime as dt + +from osmium.replication.server import OsmosisState def get_import_file(context): if context.import_file is not None: @@ -32,14 +38,18 @@ def run_osm2pgsql(context, output): - assert output in ('flex', 'pgsql', 'gazetteer', 'none') + assert output in ('flex', 'pgsql', 'gazetteer', 'null', 'nooutput') cmdline = [str(Path(context.config.userdata['BINARY']).resolve())] - cmdline.extend(('-O', output)) + + if output != 'nooutput': + cmdline.extend(('-O', output)) + cmdline.extend(context.osm2pgsql_params) + # convert table items to CLI arguments and inject constants to placeholders if context.table: - cmdline.extend(f for f in context.table.headings if f) + cmdline.extend(f.format(**context.config.userdata) for f in context.table.headings if f) for row in context.table: cmdline.extend(f.format(**context.config.userdata) for f in row if f) @@ -72,6 +82,39 @@ return proc.returncode + +def run_osm2pgsql_replication(context): + cmdline = [] + # convert table items to CLI arguments and inject constants to placeholders + if context.table: + cmdline.extend(f.format(**context.config.userdata) for f in context.table.headings if f) + for row in context.table: + cmdline.extend(f.format(**context.config.userdata) for f in row if f) + + if '-d' not in cmdline and '--database' not in cmdline: + cmdline.extend(('-d', context.config.userdata['TEST_DB'])) + + if cmdline[0] == 'update': + cmdline.extend(('--osm2pgsql-cmd', + str(Path(context.config.userdata['BINARY']).resolve()))) + + if '--' not in cmdline: + cmdline.extend(('--', '-S', str(context.default_data_dir / 'default.style'))) + + + serr = StringIO() + log_handler = logging.StreamHandler(serr) + context.osm2pgsql_replication.LOG.addHandler(log_handler) + with contextlib.redirect_stdout(StringIO()) as sout: + retval = context.osm2pgsql_replication.main(cmdline) + context.osm2pgsql_replication.LOG.removeHandler(log_handler) + + context.osm2pgsql_outdata = [sout.getvalue(), serr.getvalue()] + print(context.osm2pgsql_outdata) + + return retval + + @given("no lua tagtransform") def do_not_setup_tagtransform(context): pass @@ -106,7 +149,7 @@ @when("running osm2pgsql (?P\w+)(?: with parameters)?") -def execute_osm2pgsql_sucessfully(context, output): +def execute_osm2pgsql_successfully(context, output): returncode = run_osm2pgsql(context, output) if context.scenario.status == "skipped": @@ -127,6 +170,26 @@ assert returncode != 0, "osm2pgsql unexpectedly succeeded" +@when("running osm2pgsql-replication") +def execute_osm2pgsql_replication_successfully(context): + returncode = run_osm2pgsql_replication(context) + + assert returncode == 0,\ + f"osm2pgsql-replication failed with error code {returncode}.\n"\ + f"Output:\n{context.osm2pgsql_outdata[0]}\n{context.osm2pgsql_outdata[1]}\n" + + +@then("running osm2pgsql-replication fails(?: with returncode (?P\d+))?") +def execute_osm2pgsql_replication_successfully(context, expected): + returncode = run_osm2pgsql_replication(context) + + assert returncode != 0, "osm2pgsql-replication unexpectedly succeeded" + if expected: + assert returncode == int(expected), \ + f"osm2pgsql-replication failed with returncode {returncode} instead of {expected}."\ + f"Output:\n{context.osm2pgsql_outdata[0]}\n{context.osm2pgsql_outdata[1]}\n" + + @then("the (?P\w+) output contains") def check_program_output(context, kind): if kind == 'error': @@ -141,3 +204,18 @@ if line: assert line in s,\ f"Output '{line}' not found in {kind} output:\n{s}\n" + + +@given("the replication service at (?P.*)") +def setup_replication_mock(context, base_url): + context.osm2pgsql_replication.ReplicationServer.expected_base_url = base_url + if context.table: + context.osm2pgsql_replication.ReplicationServer.state_infos =\ + [OsmosisState(int(row[0]), + dt.datetime.strptime(row[1], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=dt.timezone.utc)) + for row in context.table] + + +@given("the URL (?P.*) returns") +def mock_url_response(context, base_url): + context.urlrequest_responses[base_url] = context.text diff -Nru osm2pgsql-1.8.1+ds/tests/CMakeLists.txt osm2pgsql-1.9.0+ds/tests/CMakeLists.txt --- osm2pgsql-1.8.1+ds/tests/CMakeLists.txt 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/CMakeLists.txt 2023-08-15 19:18:18.000000000 +0000 @@ -36,6 +36,7 @@ add_definitions(-DOSM2PGSQLDATA_DIR=\"${osm2pgsql_SOURCE_DIR}/\") add_library(catch_main_lib STATIC catch-main.cpp) +target_compile_features(catch_main_lib PUBLIC cxx_std_17) set_test(test-check-input LABELS NoDB) set_test(test-db-copy-thread) @@ -62,6 +63,7 @@ set_test(test-options-parse LABELS NoDB) set_test(test-options-projection) set_test(test-ordered-index LABELS NoDB) +set_test(test-osm-file-parsing LABELS NoDB) set_test(test-output-gazetteer) set_test(test-output-pgsql) set_test(test-output-pgsql-area) @@ -72,10 +74,10 @@ set_test(test-output-pgsql-tablespace LABELS Tablespace) set_test(test-output-pgsql-validgeom) set_test(test-output-pgsql-z_order) -set_test(test-parse-osmium LABELS NoDB) set_test(test-persistent-cache LABELS NoDB) set_test(test-pgsql) set_test(test-pgsql-capabilities) +set_test(test-properties) set_test(test-reprojection LABELS NoDB) set_test(test-taginfo LABELS NoDB) set_test(test-tile LABELS NoDB) @@ -101,7 +103,7 @@ set_test(test-output-flex-validgeom) set_test(test-output-flex-example-configs) - set(FLEX_EXAMPLE_CONFIGS "addresses,attributes,bbox,compatible,data-types,generic,geometries,indexes,places,route-relations,simple,unitable") + set(FLEX_EXAMPLE_CONFIGS "addresses,attributes,bbox,compatible,data-types,expire,generic,geometries,indexes,places,route-relations,simple,unitable") # with-schema.lua is not tested because it needs the schema created in the database set_tests_properties(test-output-flex-example-configs PROPERTIES ENVIRONMENT "EXAMPLE_FILES=${FLEX_EXAMPLE_CONFIGS}") endif() diff -Nru osm2pgsql-1.8.1+ds/tests/common-options.hpp osm2pgsql-1.9.0+ds/tests/common-options.hpp --- osm2pgsql-1.8.1+ds/tests/common-options.hpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/common-options.hpp 2023-08-15 19:18:18.000000000 +0000 @@ -28,6 +28,8 @@ m_opt.cache = 2; m_opt.append = false; m_opt.projection = reprojection::create_projection(PROJ_SPHERE_MERC); + m_opt.middle_dbschema = "public"; + m_opt.output_dbschema = "public"; } operator options_t() const { return m_opt; } Binary files /tmp/tmp4uzvfz0d/mwPfe8t7mQ/osm2pgsql-1.8.1+ds/tests/data/liechtenstein-2013-08-03.osm.pbf and /tmp/tmp4uzvfz0d/CZXBLasklI/osm2pgsql-1.9.0+ds/tests/data/liechtenstein-2013-08-03.osm.pbf differ diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_multipolygon_diff.osc osm2pgsql-1.9.0+ds/tests/data/test_multipolygon_diff.osc --- osm2pgsql-1.8.1+ds/tests/data/test_multipolygon_diff.osc 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_multipolygon_diff.osc 2023-08-15 19:18:18.000000000 +0000 @@ -28,10 +28,7 @@ - - - - + @@ -41,13 +38,7 @@ - - - - - - - + @@ -134,13 +125,7 @@ - - - - - - - + diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_output_flex_copy.lua osm2pgsql-1.9.0+ds/tests/data/test_output_flex_copy.lua --- osm2pgsql-1.8.1+ds/tests/data/test_output_flex_copy.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_output_flex_copy.lua 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,133 @@ + +local tables = {} + +tables.point = osm2pgsql.define_node_table('osm2pgsql_test_point', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'point' }, +}) + +tables.line = osm2pgsql.define_table{ + name = 'osm2pgsql_test_line', + ids = { type = 'way', id_column = 'osm_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'name', type = 'text' }, + { column = 'geom', type = 'linestring' }, + }, + cluster = 'auto' +} + +tables.polygon = osm2pgsql.define_table{ + name = 'osm2pgsql_test_polygon', + ids = { type = 'area', id_column = 'osm_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry' }, + { column = 'area', type = 'area' }, + } +} + +tables.route = osm2pgsql.define_table{ + name = 'osm2pgsql_test_route', + ids = { type = 'relation', id_column = 'osm_id' }, + columns = { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'multilinestring' }, + } +} + +local function is_polygon(tags) + if tags.aeroway + or tags.amenity + or tags.area + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] + then + return true + else + return false + end +end + +local delete_keys = { + 'odbl', + 'created_by', + 'source' +} + +local clean_tags = osm2pgsql.make_clean_tags_func(delete_keys) + +function osm2pgsql.process_node(object) + if clean_tags(object.tags) then + return + end + + tables.point:add_row({ + tags = object.tags + }) +end + +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + + if is_polygon(object.tags) then + tables.polygon:add_row({ + tags = object.tags, + name = object.tags.name, + geom = { create = 'area' } + }) + else + tables.line:add_row({ + tags = object.tags, + name = object.tags.name + }) + end +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + + if object.tags.type == 'multipolygon' or object.tags.type == 'boundary' then + tables.polygon:add_row({ + tags = object.tags, + name = object.tags.name, + geom = { create = 'area', split_at = 'multi' } + }) + return + end + + if object.tags.type == 'route' then + tables.route:add_row({ + tags = object.tags, + geom = { create = 'line' } + }) + end +end + diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_output_flex.lua osm2pgsql-1.9.0+ds/tests/data/test_output_flex.lua --- osm2pgsql-1.8.1+ds/tests/data/test_output_flex.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_output_flex.lua 2023-08-15 19:18:18.000000000 +0000 @@ -37,7 +37,7 @@ } } -function is_polygon(tags) +local function is_polygon(tags) if tags.aeroway or tags.amenity or tags.area diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_output_flex_stage2_alt.lua osm2pgsql-1.9.0+ds/tests/data/test_output_flex_stage2_alt.lua --- osm2pgsql-1.8.1+ds/tests/data/test_output_flex_stage2_alt.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_output_flex_stage2_alt.lua 2023-08-15 19:18:18.000000000 +0000 @@ -36,7 +36,7 @@ local d = w2r[object.id] if d then local refs = {} - for rel_id, rel_ref in pairs(d) do + for _, rel_ref in pairs(d) do refs[#refs + 1] = rel_ref end table.sort(refs) diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_output_flex_stage2.lua osm2pgsql-1.9.0+ds/tests/data/test_output_flex_stage2.lua --- osm2pgsql-1.8.1+ds/tests/data/test_output_flex_stage2.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_output_flex_stage2.lua 2023-08-15 19:18:18.000000000 +0000 @@ -32,7 +32,7 @@ local d = w2r[object.id] if d then local refs = {} - for rel_id, rel_ref in pairs(d) do + for _, rel_ref in pairs(d) do refs[#refs + 1] = rel_ref end table.sort(refs) diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_output_flex_uni.lua osm2pgsql-1.9.0+ds/tests/data/test_output_flex_uni.lua --- osm2pgsql-1.8.1+ds/tests/data/test_output_flex_uni.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_output_flex_uni.lua 2023-08-15 19:18:18.000000000 +0000 @@ -20,7 +20,7 @@ } } -function is_empty(some_table) +local function is_empty(some_table) return next(some_table) == nil end diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_output_flex_validgeom.lua osm2pgsql-1.9.0+ds/tests/data/test_output_flex_validgeom.lua --- osm2pgsql-1.8.1+ds/tests/data/test_output_flex_validgeom.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_output_flex_validgeom.lua 2023-08-15 19:18:18.000000000 +0000 @@ -7,7 +7,7 @@ } } -function is_empty(some_table) +local function is_empty(some_table) return next(some_table) == nil end diff -Nru osm2pgsql-1.8.1+ds/tests/data/test_output_flex_way.lua osm2pgsql-1.9.0+ds/tests/data/test_output_flex_way.lua --- osm2pgsql-1.8.1+ds/tests/data/test_output_flex_way.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/data/test_output_flex_way.lua 2023-08-15 19:18:18.000000000 +0000 @@ -20,7 +20,7 @@ local w2r = {} -function get_ids(data) +local function get_ids(data) if data then local ids = {} for rel_id, _ in pairs(data) do @@ -59,7 +59,7 @@ end end -function way_member_ids(relation) +local function way_member_ids(relation) local ids = {} for _, member in ipairs(relation.members) do if member.type == 'w' and member.role == 'mark' then diff -Nru osm2pgsql-1.8.1+ds/tests/lua/tests.lua osm2pgsql-1.9.0+ds/tests/lua/tests.lua --- osm2pgsql-1.8.1+ds/tests/lua/tests.lua 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/lua/tests.lua 2023-08-15 19:18:18.000000000 +0000 @@ -93,7 +93,7 @@ assert(tags['source:vs:source'] == nil) assert(tags['with:source:infix'] == 'keepme') - num = 0 + local num = 0 for k, v in pairs(tags) do num = num + 1 end @@ -116,7 +116,7 @@ -- split_unit -v, u = o.split_unit('20m', '') +local v, u = o.split_unit('20m', '') assert(v == 20 and u == 'm') v, u = o.split_unit('20 m') @@ -147,16 +147,16 @@ assert(v == -20000 and u == 'leagues') v, u = o.split_unit('20xx20', '') -assert(v == nil) +assert(v == nil and u == nil) v, u = o.split_unit('20-20', '') -assert(v == nil) +assert(v == nil and u == nil) v, u = o.split_unit('20xx20', 'foo') -assert(v == nil) +assert(v == nil and u == nil) v, u = o.split_unit('abc', 'def') -assert(v == nil) +assert(v == nil and u == nil) v, u = o.split_unit(nil) assert(v == nil and u == nil) @@ -166,7 +166,7 @@ -- split_string -r = o.split_string('ab c;d;e f;ghi') +local r = o.split_string('ab c;d;e f;ghi') assert(#r == 4) assert(r[1] == 'ab c') assert(r[2] == 'd') diff -Nru osm2pgsql-1.8.1+ds/tests/test-db-copy-mgr.cpp osm2pgsql-1.9.0+ds/tests/test-db-copy-mgr.cpp --- osm2pgsql-1.8.1+ds/tests/test-db-copy-mgr.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-db-copy-mgr.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -27,9 +27,8 @@ conn.exec("CREATE TABLE test_copy_mgr (id int8{}{})", cols.empty() ? "" : ",", cols); - auto table = std::make_shared(); - table->name = "test_copy_mgr"; - table->id = "id"; + auto table = + std::make_shared("public", "test_copy_mgr", "id"); return table; } @@ -88,153 +87,165 @@ } } -TEST_CASE("copy_mgr_t") +TEST_CASE("copy_mgr_t: Insert null") { copy_mgr_t mgr{std::make_shared(db.conninfo())}; - SECTION("Insert null") - { - auto const t = setup_table("big int8, t text"); + auto const t = setup_table("big int8, t text"); - mgr.new_line(t); - mgr.add_column(0); - mgr.add_null_column(); - mgr.add_null_column(); - mgr.finish_line(); - mgr.sync(); + mgr.new_line(t); + mgr.add_column(0); + mgr.add_null_column(); + mgr.add_null_column(); + mgr.finish_line(); + mgr.sync(); - auto const conn = db.connect(); - auto const res = conn.require_row("SELECT * FROM test_copy_mgr"); + auto const conn = db.connect(); + auto const res = conn.require_row("SELECT * FROM test_copy_mgr"); - CHECK(res.is_null(0, 1)); - CHECK(res.is_null(0, 2)); - } + CHECK(res.is_null(0, 1)); + CHECK(res.is_null(0, 2)); +} - SECTION("Insert numbers") - { - auto const t = setup_table("big int8, small smallint"); +TEST_CASE("copy_mgr_t: Insert numbers") +{ + copy_mgr_t mgr{std::make_shared(db.conninfo())}; - add_row(&mgr, t, 34, 0xfff12345678ULL, -4457); - check_row({"34", "17588196497016", "-4457"}); - } + auto const t = setup_table("big int8, small smallint"); - SECTION("Insert strings") - { - auto const t = setup_table("s0 text, s1 varchar"); + add_row(&mgr, t, 34, 0xfff12345678ULL, -4457); + check_row({"34", "17588196497016", "-4457"}); +} - SECTION("Simple strings") - { - add_row(&mgr, t, -2, "foo", "l"); - check_row({"-2", "foo", "l"}); - } - - SECTION("Strings with special characters") - { - add_row(&mgr, t, -2, "va\tr", "meme\n"); - check_row({"-2", "va\tr", "meme\n"}); - } - - SECTION("Strings with more special characters") - { - add_row(&mgr, t, -2, "\rrun", "K\\P"); - check_row({"-2", "\rrun", "K\\P"}); - } - - SECTION("Strings with space and quote") - { - add_row(&mgr, t, 1, "with space", "name \"quoted\""); - check_row({"1", "with space", "name \"quoted\""}); - } - } +TEST_CASE("copy_mgr_t: Insert strings") +{ + copy_mgr_t mgr{std::make_shared(db.conninfo())}; - SECTION("Insert int arrays") - { - auto const t = setup_table("a int[]"); + auto const t = setup_table("s0 text, s1 varchar"); - add_array(&mgr, t, -9000, {45, -2, 0, 56}); - check_row({"-9000", "{45,-2,0,56}"}); + SECTION("Simple strings") + { + add_row(&mgr, t, -2, "foo", "l"); + check_row({"-2", "foo", "l"}); } - SECTION("Insert string arrays") + SECTION("Strings with special characters") { - auto const t = setup_table("a text[]"); + add_row(&mgr, t, -2, "va\tr", "meme\n"); + check_row({"-2", "va\tr", "meme\n"}); + } - add_array(&mgr, t, 3, - {"foo", "", "with space", "with \"quote\"", - "the\t", "line\nbreak", "rr\rrr", "s\\l"}); - check_row({"3", "{foo,\"\",\"with space\",\"with " - "\\\"quote\\\"\",\"the\t\",\"line\nbreak\"," - "\"rr\rrr\",\"s\\\\l\"}"}); - - auto const c = db.connect(); - CHECK(c.result_as_string("SELECT a[4] FROM test_copy_mgr") == - "with \"quote\""); - CHECK(c.result_as_string("SELECT a[5] FROM test_copy_mgr") == "the\t"); - CHECK(c.result_as_string("SELECT a[6] FROM test_copy_mgr") == - "line\nbreak"); - CHECK(c.result_as_string("SELECT a[7] FROM test_copy_mgr") == "rr\rrr"); - CHECK(c.result_as_string("SELECT a[8] FROM test_copy_mgr") == "s\\l"); + SECTION("Strings with more special characters") + { + add_row(&mgr, t, -2, "\rrun", "K\\P"); + check_row({"-2", "\rrun", "K\\P"}); } - SECTION("Insert hashes") + SECTION("Strings with space and quote") { - auto const t = setup_table("h hstore"); + add_row(&mgr, t, 1, "with space", "name \"quoted\""); + check_row({"1", "with space", "name \"quoted\""}); + } +} - std::vector> const values = { - {"one", "two"}, {"key 1", "value 1"}, - {"\"key\"", "\"value\""}, {"key\t2", "value\t2"}, - {"key\n3", "value\n3"}, {"key\r4", "value\r4"}, - {"key\\5", "value\\5"}}; +TEST_CASE("copy_mgr_t: Insert int arrays") +{ + copy_mgr_t mgr{std::make_shared(db.conninfo())}; - add_hash(&mgr, t, 42, values); + auto const t = setup_table("a int[]"); - auto const c = db.connect(); + add_array(&mgr, t, -9000, {45, -2, 0, 56}); + check_row({"-9000", "{45,-2,0,56}"}); +} - for (auto const &[k, v] : values) { - auto const res = c.result_as_string( - fmt::format("SELECT h->'{}' FROM test_copy_mgr", k)); - CHECK(res == v); - } - } +TEST_CASE("copy_mgr_t: Insert string arrays") +{ + copy_mgr_t mgr{std::make_shared(db.conninfo())}; - SECTION("Insert something and roll back") - { - auto const t = setup_table("t text"); + auto const t = setup_table("a text[]"); - mgr.new_line(t); - mgr.add_column(0); - mgr.add_column("foo"); - mgr.rollback_line(); - mgr.sync(); + add_array(&mgr, t, 3, + {"foo", "", "with space", "with \"quote\"", "the\t", + "line\nbreak", "rr\rrr", "s\\l"}); + check_row({"3", "{foo,\"\",\"with space\",\"with " + "\\\"quote\\\"\",\"the\t\",\"line\nbreak\"," + "\"rr\rrr\",\"s\\\\l\"}"}); + + auto const c = db.connect(); + CHECK(c.result_as_string("SELECT a[4] FROM test_copy_mgr") == + "with \"quote\""); + CHECK(c.result_as_string("SELECT a[5] FROM test_copy_mgr") == "the\t"); + CHECK(c.result_as_string("SELECT a[6] FROM test_copy_mgr") == + "line\nbreak"); + CHECK(c.result_as_string("SELECT a[7] FROM test_copy_mgr") == "rr\rrr"); + CHECK(c.result_as_string("SELECT a[8] FROM test_copy_mgr") == "s\\l"); +} - auto const conn = db.connect(); - CHECK(conn.get_count("test_copy_mgr") == 0); - } +TEST_CASE("copy_mgr_t: Insert hashes") +{ + copy_mgr_t mgr{std::make_shared(db.conninfo())}; - SECTION("Insert something, insert more, roll back, insert something else") - { - auto const t = setup_table("t text"); + auto const t = setup_table("h hstore"); + + std::vector> const values = { + {"one", "two"}, {"key 1", "value 1"}, + {"\"key\"", "\"value\""}, {"key\t2", "value\t2"}, + {"key\n3", "value\n3"}, {"key\r4", "value\r4"}, + {"key\\5", "value\\5"}}; - mgr.new_line(t); - mgr.add_column(0); - mgr.add_column("good"); - mgr.finish_line(); - - mgr.new_line(t); - mgr.add_column(1); - mgr.add_column("bad"); - mgr.rollback_line(); - - mgr.new_line(t); - mgr.add_column(2); - mgr.add_column("better"); - mgr.finish_line(); - mgr.sync(); - - auto const conn = db.connect(); - auto const res = conn.exec("SELECT t FROM test_copy_mgr ORDER BY id"); - CHECK(res.num_tuples() == 2); - CHECK(res.get(0, 0) == "good"); - CHECK(res.get(1, 0) == "better"); + add_hash(&mgr, t, 42, values); + + auto const c = db.connect(); + + for (auto const &[k, v] : values) { + auto const res = c.result_as_string( + fmt::format("SELECT h->'{}' FROM test_copy_mgr", k)); + CHECK(res == v); } } + +TEST_CASE("copy_mgr_t: Insert something and roll back") +{ + copy_mgr_t mgr{std::make_shared(db.conninfo())}; + + auto const t = setup_table("t text"); + + mgr.new_line(t); + mgr.add_column(0); + mgr.add_column("foo"); + mgr.rollback_line(); + mgr.sync(); + + auto const conn = db.connect(); + CHECK(conn.get_count("test_copy_mgr") == 0); +} + +TEST_CASE("copy_mgr_t: Insert something, insert more, roll back, insert " + "something else") +{ + copy_mgr_t mgr{std::make_shared(db.conninfo())}; + + auto const t = setup_table("t text"); + + mgr.new_line(t); + mgr.add_column(0); + mgr.add_column("good"); + mgr.finish_line(); + + mgr.new_line(t); + mgr.add_column(1); + mgr.add_column("bad"); + mgr.rollback_line(); + + mgr.new_line(t); + mgr.add_column(2); + mgr.add_column("better"); + mgr.finish_line(); + mgr.sync(); + + auto const conn = db.connect(); + auto const res = conn.exec("SELECT t FROM test_copy_mgr ORDER BY id"); + CHECK(res.num_tuples() == 2); + CHECK(res.get(0, 0) == "good"); + CHECK(res.get(1, 0) == "better"); +} diff -Nru osm2pgsql-1.8.1+ds/tests/test-db-copy-thread.cpp osm2pgsql-1.9.0+ds/tests/test-db-copy-thread.cpp --- osm2pgsql-1.8.1+ds/tests/test-db-copy-thread.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-db-copy-thread.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -27,9 +27,8 @@ conn.exec("DROP TABLE IF EXISTS test_copy_thread"); conn.exec("CREATE TABLE test_copy_thread (id int8)"); - auto const table = std::make_shared(); - table->name = "test_copy_thread"; - table->id = "id"; + auto const table = + std::make_shared("public", "test_copy_thread", "id"); db_copy_thread_t t(db.conninfo()); using cmd_copy_t = db_cmd_copy_delete_t; @@ -157,9 +156,8 @@ "osm_id bigint," "class text)"); - auto table = std::make_shared(); - table->name = "test_copy_thread"; - table->id = "place_id"; + auto table = std::make_shared( + "public", "test_copy_thread", "place_id"); db_copy_thread_t t(db.conninfo()); using cmd_copy_t = db_cmd_copy_delete_t; diff -Nru osm2pgsql-1.8.1+ds/tests/test-flex-indexes.cpp osm2pgsql-1.9.0+ds/tests/test-flex-indexes.cpp --- osm2pgsql-1.8.1+ds/tests/test-flex-indexes.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-flex-indexes.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -11,9 +11,10 @@ #include "flex-lua-index.hpp" #include "flex-table.hpp" -#include "lua.hpp" #include "pgsql-capabilities-int.hpp" +#include + class test_framework { public: @@ -47,7 +48,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("geom", "geometry", ""); REQUIRE(table.indexes().empty()); @@ -70,7 +71,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("a", "int", ""); table.add_column("b", "int", ""); @@ -92,7 +93,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("col", "int", ""); REQUIRE(tf.run_lua( @@ -114,7 +115,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.set_index_tablespace("foo"); table.add_column("col", "int", ""); @@ -136,7 +137,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("col", "int", ""); REQUIRE(tf.run_lua("return { method = 'btree', column = 'col', tablespace " @@ -158,7 +159,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("col", "text", ""); REQUIRE(tf.run_lua("return { method = 'btree', expression = 'lower(col)'," @@ -180,7 +181,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("col", "int", ""); table.add_column("extra", "int", ""); @@ -203,7 +204,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("col", "int", ""); table.add_column("extra", "int", ""); @@ -226,7 +227,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("col", "int", ""); table.add_column("extra", "int", ""); @@ -249,7 +250,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("a", "int", ""); table.add_column("b", "int", ""); @@ -272,7 +273,7 @@ { test_framework const tf; - flex_table_t table{"test_table"}; + flex_table_t table{"public", "test_table"}; table.add_column("col", "text", ""); SECTION("empty index description") { REQUIRE(tf.run_lua("return {}")); } diff -Nru osm2pgsql-1.8.1+ds/tests/test-geom-multipolygons.cpp osm2pgsql-1.9.0+ds/tests/test-geom-multipolygons.cpp --- osm2pgsql-1.8.1+ds/tests/test-geom-multipolygons.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-geom-multipolygons.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -21,6 +21,7 @@ geom::geometry_t geom{geom::multipolygon_t{}}; auto &mp = geom.get(); + // MULTIPOLYGON(((0 0, 0 1, 1 1, 1 0, 0 0))) mp.add_geometry( geom::polygon_t{geom::ring_t{{0, 0}, {0, 1}, {1, 1}, {1, 0}, {0, 0}}}); @@ -28,6 +29,7 @@ REQUIRE(dimension(geom) == 2); REQUIRE(num_geometries(geom) == 1); REQUIRE(area(geom) == Approx(1.0)); + REQUIRE(spherical_area(geom) == Approx(12364031798.5)); REQUIRE(length(geom) == Approx(0.0)); REQUIRE(centroid(geom) == geom::geometry_t{geom::point_t{0.5, 0.5}}); REQUIRE(geometry_n(geom, 1) == @@ -40,6 +42,7 @@ geom::geometry_t geom{geom::multipolygon_t{}}; auto &mp = geom.get(); + // MULTIPOLYGON(((0 0, 0 1, 1 1, 1 0, 0 0)), ((2 2, 2 5, 5 5, 5 2, 2 2), (3 3, 4 3, 4 4, 3 4, 3 3))) mp.add_geometry( geom::polygon_t{geom::ring_t{{0, 0}, {0, 1}, {1, 1}, {1, 0}, {0, 0}}}); @@ -56,6 +59,7 @@ REQUIRE(dimension(geom) == 2); REQUIRE(num_geometries(geom) == 2); REQUIRE(area(geom) == Approx(9.0)); + REQUIRE(spherical_area(geom) == Approx(111106540105.7)); REQUIRE(length(geom) == Approx(0.0)); } diff -Nru osm2pgsql-1.8.1+ds/tests/test-geom-points.cpp osm2pgsql-1.9.0+ds/tests/test-geom-points.cpp --- osm2pgsql-1.8.1+ds/tests/test-geom-points.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-geom-points.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -61,6 +61,24 @@ REQUIRE(geom.get() == geom::point_t{1.1, 2.2}); } +TEST_CASE("point order", "[NoDB]") +{ + geom::point_t const p{10, 10}; + REQUIRE_FALSE(p < p); + REQUIRE_FALSE(p > p); + + std::vector points = { + {10, 10}, {20, 10}, {13, 14}, {13, 10} + }; + + std::sort(points.begin(), points.end()); + + REQUIRE(points[0] == geom::point_t(10, 10)); + REQUIRE(points[1] == geom::point_t(13, 10)); + REQUIRE(points[2] == geom::point_t(13, 14)); + REQUIRE(points[3] == geom::point_t(20, 10)); +} + TEST_CASE("geom::distance", "[NoDB]") { geom::point_t const p1{10, 10}; diff -Nru osm2pgsql-1.8.1+ds/tests/test-geom-polygons.cpp osm2pgsql-1.9.0+ds/tests/test-geom-polygons.cpp --- osm2pgsql-1.8.1+ds/tests/test-geom-polygons.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-geom-polygons.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -18,12 +18,14 @@ TEST_CASE("polygon geometry without inner", "[NoDB]") { + // POLYGON((0 0, 0 1, 1 1, 1 0, 0 0)) geom::geometry_t const geom{ geom::polygon_t{geom::ring_t{{0, 0}, {0, 1}, {1, 1}, {1, 0}, {0, 0}}}}; REQUIRE(dimension(geom) == 2); REQUIRE(num_geometries(geom) == 1); REQUIRE(area(geom) == Approx(1.0)); + REQUIRE(spherical_area(geom) == Approx(12364031798.5)); REQUIRE(length(geom) == Approx(0.0)); REQUIRE(geometry_type(geom) == "POLYGON"); REQUIRE(centroid(geom) == geom::geometry_t{geom::point_t{0.5, 0.5}}); @@ -32,12 +34,14 @@ TEST_CASE("polygon geometry without inner (reverse)", "[NoDB]") { + // POLYGON((0 0, 1 0, 1 1, 0 1, 0 0)) geom::geometry_t const geom{ geom::polygon_t{geom::ring_t{{0, 0}, {1, 0}, {1, 1}, {0, 1}, {0, 0}}}}; REQUIRE(dimension(geom) == 2); REQUIRE(num_geometries(geom) == 1); REQUIRE(area(geom) == Approx(1.0)); + REQUIRE(spherical_area(geom) == Approx(12364031798.5)); REQUIRE(length(geom) == Approx(0.0)); REQUIRE(geometry_type(geom) == "POLYGON"); REQUIRE(centroid(geom) == geom::geometry_t{geom::point_t{0.5, 0.5}}); @@ -48,6 +52,8 @@ geom::polygon_t polygon; REQUIRE(polygon.outer().empty()); + + // POLYGON((0 0, 0 3, 3 3, 3 0, 0 0), (1 1, 2 1, 2 2, 1 2, 1 1)) polygon.outer() = geom::ring_t{{0, 0}, {0, 3}, {3, 3}, {3, 0}, {0, 0}}; polygon.inners().emplace_back( geom::ring_t{{1, 1}, {2, 1}, {2, 2}, {1, 2}, {1, 1}}); @@ -59,6 +65,7 @@ REQUIRE(dimension(geom) == 2); REQUIRE(num_geometries(geom) == 1); REQUIRE(area(geom) == Approx(8.0)); + REQUIRE(spherical_area(geom) == Approx(98893356298.4)); REQUIRE(length(geom) == Approx(0.0)); REQUIRE(geometry_type(geom) == "POLYGON"); REQUIRE(centroid(geom) == geom::geometry_t{geom::point_t{1.5, 1.5}}); diff -Nru osm2pgsql-1.8.1+ds/tests/test-lua-utils.cpp osm2pgsql-1.9.0+ds/tests/test-lua-utils.cpp --- osm2pgsql-1.8.1+ds/tests/test-lua-utils.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-lua-utils.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -11,10 +11,7 @@ #include "lua-utils.hpp" -extern "C" -{ -#include -} +#include // Run the Lua code in "code" and then execute the function "func". template @@ -46,7 +43,7 @@ }); } -TEST_CASE("check luaX_is_array", "[NoDB]") +TEST_CASE("check luaX_is_array with arrays", "[NoDB]") { std::shared_ptr lua_state{ luaL_newstate(), [](lua_State *state) { lua_close(state); }}; @@ -66,6 +63,12 @@ test_lua(lua_state.get(), "return { [1] = 1, [2] = 2 }", [&](){ REQUIRE(luaX_is_array(lua_state.get())); }); +} + +TEST_CASE("check luaX_is_array with non-arrays", "[NoDB]") +{ + std::shared_ptr lua_state{ + luaL_newstate(), [](lua_State *state) { lua_close(state); }}; test_lua(lua_state.get(), "return { 1, nil, 3 }", [&](){ REQUIRE_FALSE(luaX_is_array(lua_state.get())); diff -Nru osm2pgsql-1.8.1+ds/tests/test-middle.cpp osm2pgsql-1.9.0+ds/tests/test-middle.cpp --- osm2pgsql-1.8.1+ds/tests/test-middle.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-middle.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -82,6 +82,26 @@ } }; +struct options_slim_new_format +{ + static options_t options(testing::pg::tempdb_t const &tmpdb) + { + options_t o = testing::opt_t().slim(tmpdb); + o.middle_database_format = 2; + return o; + } +}; + +struct options_slim_new_format_with_flatnodes +{ + static options_t options(testing::pg::tempdb_t const &tmpdb) + { + options_t o = testing::opt_t().slim(tmpdb).flatnodes(); + o.middle_database_format = 2; + return o; + } +}; + struct options_ram_optimized { static options_t options(testing::pg::tempdb_t const &) @@ -91,22 +111,24 @@ }; TEMPLATE_TEST_CASE("middle import", "", options_slim_default, - options_slim_with_lc_prefix, options_slim_with_uc_prefix, - options_slim_with_schema, options_ram_optimized) + options_slim_new_format, options_slim_with_lc_prefix, + options_slim_with_uc_prefix, options_slim_with_schema, + options_ram_optimized) { options_t const options = TestType::options(db); testing::cleanup::file_t const flatnode_cleaner{options.flat_node_file}; auto conn = db.connect(); auto const num_tables = - conn.get_count("pg_tables", "schemaname = 'public'"); + conn.get_count("pg_catalog.pg_tables", "schemaname = 'public'"); auto const num_indexes = - conn.get_count("pg_indexes", "schemaname = 'public'"); + conn.get_count("pg_catalog.pg_indexes", "schemaname = 'public'"); auto const num_procs = - conn.get_count("pg_proc", "pronamespace = (SELECT oid FROM " - "pg_namespace WHERE nspname = 'public')"); + conn.get_count("pg_catalog.pg_proc", + "pronamespace = (SELECT oid FROM " + "pg_catalog.pg_namespace WHERE nspname = 'public')"); - if (!options.middle_dbschema.empty()) { + if (options.middle_dbschema == "osm") { conn.exec("CREATE SCHEMA IF NOT EXISTS osm;"); } @@ -130,6 +152,8 @@ // set the node mid->node(node); mid->after_nodes(); + mid->after_ways(); + mid->after_relations(); // getting it back works only via a waylist auto &nodes = buffer.add_way("w3 Nn1234").nodes(); @@ -161,7 +185,8 @@ for (osmid_t i = 1; i <= 10; ++i) { nds.push_back(i); auto const &node = buffer.add_node(fmt::format( - "n{} x{:.7f} y{:.7f}", i, lon - i * 0.003, lat + i * 0.001)); + "n{} x{:.7f} y{:.7f}", i, lon - static_cast(i) * 0.003, + lat + static_cast(i) * 0.001)); mid->node(node); } mid->after_nodes(); @@ -169,6 +194,7 @@ // set the way mid->way(buffer.add_way(way_id, nds)); mid->after_ways(); + mid->after_relations(); // get it back osmium::memory::Buffer outbuf{4096, @@ -303,15 +329,16 @@ REQUIRE_FALSE(mid_q->relation_get(999, &outbuf)); } - if (!options.middle_dbschema.empty()) { - REQUIRE(num_tables == - conn.get_count("pg_tables", "schemaname = 'public'")); - REQUIRE(num_indexes == - conn.get_count("pg_indexes", "schemaname = 'public'")); + if (options.middle_dbschema != "public") { + REQUIRE(num_tables == conn.get_count("pg_catalog.pg_tables", + "schemaname = 'public'")); + REQUIRE(num_indexes == conn.get_count("pg_catalog.pg_indexes", + "schemaname = 'public'")); REQUIRE(num_procs == - conn.get_count("pg_proc", - "pronamespace = (SELECT oid FROM " - "pg_namespace WHERE nspname = 'public')")); + conn.get_count( + "pg_catalog.pg_proc", + "pronamespace = (SELECT oid FROM " + "pg_catalog.pg_namespace WHERE nspname = 'public')")); } } @@ -339,7 +366,9 @@ } TEMPLATE_TEST_CASE("middle: add, delete and update node", "", - options_slim_default, options_flat_node_cache) + options_slim_default, options_slim_new_format, + options_flat_node_cache, + options_slim_new_format_with_flatnodes) { auto thread_pool = std::make_shared(1U); @@ -371,6 +400,7 @@ mid->node(node10); mid->node(node11); mid->after_nodes(); + mid->after_ways(); mid->after_relations(); check_node(mid, node10); @@ -401,6 +431,7 @@ mid->node(node10d); mid->node(node42d); mid->after_nodes(); + mid->after_ways(); mid->after_relations(); REQUIRE(no_node(mid, 5)); @@ -431,6 +462,7 @@ mid->node(node12d); mid->node(node12); mid->after_nodes(); + mid->after_ways(); mid->after_relations(); check_node(mid, node10a); @@ -456,6 +488,7 @@ mid->node(node12); mid->after_nodes(); + mid->after_ways(); mid->after_relations(); REQUIRE(no_node(mid, 5)); @@ -491,6 +524,25 @@ REQUIRE(mid_q->way_get(orig_way.id(), &outbuf)); auto const &way = outbuf.get(0); + CHECK(orig_way.id() == way.id()); + CHECK(orig_way.timestamp() == way.timestamp()); + CHECK(orig_way.version() == way.version()); + CHECK(orig_way.changeset() == way.changeset()); + CHECK(orig_way.uid() == way.uid()); + CHECK(std::strcmp(orig_way.user(), way.user()) == 0); + + REQUIRE(orig_way.tags().size() == way.tags().size()); + for (auto it1 = orig_way.tags().begin(), it2 = way.tags().begin(); + it1 != orig_way.tags().end(); ++it1, ++it2) { + CHECK(*it1 == *it2); + } + + REQUIRE(orig_way.nodes().size() == way.nodes().size()); + for (auto it1 = orig_way.nodes().begin(), it2 = way.nodes().begin(); + it1 != orig_way.nodes().end(); ++it1, ++it2) { + CHECK(*it1 == *it2); + } + osmium::CRC orig_crc; orig_crc.update(orig_way); @@ -533,7 +585,9 @@ } TEMPLATE_TEST_CASE("middle: add, delete and update way", "", - options_slim_default, options_flat_node_cache) + options_slim_default, options_slim_new_format, + options_flat_node_cache, + options_slim_new_format_with_flatnodes) { auto thread_pool = std::make_shared(1U); @@ -565,6 +619,7 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); mid->way(way20); mid->way(way21); mid->after_ways(); @@ -594,6 +649,7 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); mid->way(way5d); mid->way(way20d); mid->way(way42d); @@ -623,6 +679,7 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); mid->way(way20d); mid->way(way20a); mid->way(way22d); @@ -655,6 +712,7 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); mid->way(way22); mid->after_ways(); mid->after_relations(); @@ -680,7 +738,8 @@ } TEMPLATE_TEST_CASE("middle: add way with attributes", "", options_slim_default, - options_flat_node_cache) + options_slim_new_format, options_flat_node_cache, + options_slim_new_format_with_flatnodes) { auto thread_pool = std::make_shared(1U); @@ -704,23 +763,16 @@ auto &way20_no_attr = buffer.add_way("w20 Nn10,n11 Thighway=residential,name=High_Street"); - // The same way but with attributes in tags. - // The order of the tags is important here! - auto &way20_attr_tags = buffer.add_way( - "w20 Nn10,n11 " - "Thighway=residential,name=High_Street,osm_user=,osm_uid=789," - "osm_version=123,osm_timestamp=2009-02-13T23:31:30Z,osm_changeset=456"); - { auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); mid->way(way20); mid->after_ways(); mid->after_relations(); - check_way(mid, - options.extra_attributes ? way20_attr_tags : way20_no_attr); + check_way(mid, options.extra_attributes ? way20 : way20_no_attr); } // From now on use append mode to not destroy the data we just added. @@ -730,8 +782,7 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); - check_way(mid, - options.extra_attributes ? way20_attr_tags : way20_no_attr); + check_way(mid, options.extra_attributes ? way20 : way20_no_attr); } } @@ -748,6 +799,28 @@ REQUIRE(mid_q->relation_get(orig_relation.id(), &outbuf)); auto const &relation = outbuf.get(0); + CHECK(orig_relation.id() == relation.id()); + CHECK(orig_relation.timestamp() == relation.timestamp()); + CHECK(orig_relation.version() == relation.version()); + CHECK(orig_relation.changeset() == relation.changeset()); + CHECK(orig_relation.uid() == relation.uid()); + CHECK(std::strcmp(orig_relation.user(), relation.user()) == 0); + + REQUIRE(orig_relation.tags().size() == relation.tags().size()); + for (auto it1 = orig_relation.tags().begin(), it2 = relation.tags().begin(); + it1 != orig_relation.tags().end(); ++it1, ++it2) { + CHECK(*it1 == *it2); + } + + REQUIRE(orig_relation.members().size() == relation.members().size()); + for (auto it1 = orig_relation.members().begin(), + it2 = relation.members().begin(); + it1 != orig_relation.members().end(); ++it1, ++it2) { + CHECK(it1->type() == it2->type()); + CHECK(it1->ref() == it2->ref()); + CHECK(std::strcmp(it1->role(), it2->role()) == 0); + } + osmium::CRC orig_crc; orig_crc.update(orig_relation); @@ -766,7 +839,9 @@ } TEMPLATE_TEST_CASE("middle: add, delete and update relation", "", - options_slim_default, options_flat_node_cache) + options_slim_default, options_slim_new_format, + options_flat_node_cache, + options_slim_new_format_with_flatnodes) { auto thread_pool = std::make_shared(1U); @@ -777,14 +852,14 @@ // Create some relations we'll use for the tests. test_buffer_t buffer; auto const &relation30 = buffer.add_relation( - "r30 Mw10@outer,w11@innder Ttype=multipolygon,name=Penguin_Park"); + "r30 Mw10@outer,w11@innder Tname=Penguin_Park,type=multipolygon"); auto const &relation31 = buffer.add_relation("r31 Mn10@"); auto const &relation32 = buffer.add_relation("r32 Mr39@ Ttype=site"); auto const &relation30a = buffer.add_relation( - "r30 Mw10@outer,w11@outer Ttype=multipolygon,name=Pigeon_Park"); + "r30 Mw10@outer,w11@outer Tname=Pigeon_Park,type=multipolygon"); auto const &relation5d = buffer.add_relation("r5 dD"); auto const &relation30d = buffer.add_relation("r30 dD"); @@ -798,6 +873,8 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); + mid->after_ways(); mid->relation(relation30); mid->relation(relation31); mid->after_relations(); @@ -826,6 +903,8 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); + mid->after_ways(); mid->relation(relation5d); mid->relation(relation30d); mid->relation(relation42d); @@ -854,6 +933,8 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); + mid->after_ways(); mid->relation(relation30d); mid->relation(relation30a); mid->relation(relation32d); @@ -885,6 +966,8 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); + mid->after_ways(); mid->relation(relation32); mid->after_relations(); @@ -909,7 +992,9 @@ } TEMPLATE_TEST_CASE("middle: add relation with attributes", "", - options_slim_default, options_flat_node_cache) + options_slim_default, options_slim_new_format, + options_flat_node_cache, + options_slim_new_format_with_flatnodes) { auto thread_pool = std::make_shared(1U); @@ -924,27 +1009,22 @@ test_buffer_t buffer; auto const &relation30 = buffer.add_relation( "r30 v123 c456 i789 t2009-02-13T23:31:30Z Mw10@outer,w11@inner " - "Ttype=multipolygon,name=Penguin_Park"); + "Tname=Penguin_Park,type=multipolygon"); // The same relation but with default attributes. auto const &relation30_no_attr = buffer.add_relation( - "r30 Mw10@outer,w11@inner Ttype=multipolygon,name=Penguin_Park"); - - // The same relation but with attributes in tags. - // The order of the tags is important here! - auto const &relation30_attr_tags = buffer.add_relation( - "r30 Mw10@outer,w11@inner " - "Ttype=multipolygon,name=Penguin_Park,osm_user=,osm_uid=789," - "osm_version=123,osm_timestamp=2009-02-13T23:31:30Z,osm_changeset=456"); + "r30 Mw10@outer,w11@inner Tname=Penguin_Park,type=multipolygon"); { auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); + mid->after_ways(); mid->relation(relation30); mid->after_relations(); - check_relation(mid, options.extra_attributes ? relation30_attr_tags + check_relation(mid, options.extra_attributes ? relation30 : relation30_no_attr); } @@ -955,13 +1035,14 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); - check_relation(mid, options.extra_attributes ? relation30_attr_tags + check_relation(mid, options.extra_attributes ? relation30 : relation30_no_attr); } } TEMPLATE_TEST_CASE("middle: change nodes in way", "", options_slim_default, - options_flat_node_cache) + options_slim_new_format, options_flat_node_cache, + options_slim_new_format_with_flatnodes) { auto thread_pool = std::make_shared(1U); @@ -1011,6 +1092,9 @@ check_way_nodes(mid, way21.id(), {&node11, &node12}); REQUIRE_FALSE(dependency_manager.has_pending()); + + mid->stop(); + mid->wait(); } // From now on use append mode to not destroy the data we just added. @@ -1026,6 +1110,10 @@ mid->node(node10a); dependency_manager.node_changed(10); mid->after_nodes(); + dependency_manager.after_nodes(); + mid->after_ways(); + dependency_manager.after_ways(); + mid->after_relations(); REQUIRE(dependency_manager.has_pending()); idlist_t const way_ids = dependency_manager.get_pending_way_ids(); @@ -1041,9 +1129,11 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); mid->way(way22); mid->after_ways(); mid->after_relations(); + check_way(mid, way22); } { @@ -1055,6 +1145,10 @@ mid->node(node10a); dependency_manager.node_changed(10); mid->after_nodes(); + dependency_manager.after_nodes(); + mid->after_ways(); + dependency_manager.after_ways(); + mid->after_relations(); REQUIRE(dependency_manager.has_pending()); idlist_t const way_ids = dependency_manager.get_pending_way_ids(); @@ -1073,6 +1167,7 @@ auto mid = std::make_shared(thread_pool, &options); mid->start(); + mid->after_nodes(); mid->way(way20d); mid->way(way20a); mid->after_ways(); @@ -1091,6 +1186,10 @@ mid->node(node10a); dependency_manager.node_changed(10); mid->after_nodes(); + dependency_manager.after_nodes(); + mid->after_ways(); + dependency_manager.after_ways(); + mid->after_relations(); REQUIRE_FALSE(dependency_manager.has_pending()); } @@ -1098,7 +1197,8 @@ } TEMPLATE_TEST_CASE("middle: change nodes in relation", "", options_slim_default, - options_flat_node_cache) + options_slim_new_format, options_flat_node_cache, + options_slim_new_format_with_flatnodes) { auto thread_pool = std::make_shared(1U); @@ -1138,6 +1238,9 @@ mid->relation(rel30); mid->relation(rel31); mid->after_relations(); + + mid->stop(); + mid->wait(); } // From now on use append mode to not destroy the data we just added. @@ -1153,6 +1256,9 @@ mid->node(node10a); dependency_manager.node_changed(10); mid->after_nodes(); + dependency_manager.after_nodes(); + mid->after_ways(); + dependency_manager.after_ways(); mid->after_relations(); REQUIRE(dependency_manager.has_pending()); @@ -1172,6 +1278,9 @@ mid->node(node11a); dependency_manager.node_changed(11); mid->after_nodes(); + dependency_manager.after_nodes(); + mid->after_ways(); + dependency_manager.after_ways(); mid->after_relations(); REQUIRE(dependency_manager.has_pending()); diff -Nru osm2pgsql-1.8.1+ds/tests/test-node-locations.cpp osm2pgsql-1.9.0+ds/tests/test-node-locations.cpp --- osm2pgsql-1.8.1+ds/tests/test-node-locations.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-node-locations.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -64,7 +64,8 @@ } for (osmid_t id = 1; id <= max_id; ++id) { - nl.set(id, {id + 0.1, id + 0.2}); + nl.set(id, + {static_cast(id) + 0.1, static_cast(id) + 0.2}); } REQUIRE(static_cast(nl.size()) == max_id); diff -Nru osm2pgsql-1.8.1+ds/tests/test-options-parse.cpp osm2pgsql-1.9.0+ds/tests/test-options-parse.cpp --- osm2pgsql-1.8.1+ds/tests/test-options-parse.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-options-parse.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -11,7 +11,7 @@ #include -#include "options.hpp" +#include "command-line-parser.hpp" #include "taginfo-impl.hpp" #include "tagtransform.hpp" @@ -21,22 +21,27 @@ { opts.insert(opts.begin(), "osm2pgsql"); opts.push_back(TEST_PBF); - REQUIRE_THROWS_WITH(options_t((int)opts.size(), (char **)opts.data()), - Catch::Matchers::Contains(msg)); + + REQUIRE_THROWS_WITH( + parse_command_line((int)opts.size(), (char **)opts.data()), + Catch::Matchers::Contains(msg)); } static options_t opt(std::vector opts) { opts.insert(opts.begin(), "osm2pgsql"); opts.push_back(TEST_PBF); - return {(int)opts.size(), (char **)opts.data()}; + + return parse_command_line((int)opts.size(), (char **)opts.data()); } TEST_CASE("Insufficient arguments", "[NoDB]") { std::vector opts = {"osm2pgsql", "-a", "-c", "--slim"}; - REQUIRE_THROWS_WITH(options_t((int)opts.size(), (char **)opts.data()), - Catch::Matchers::Contains("Missing input")); + + REQUIRE_THROWS_WITH( + parse_command_line((int)opts.size(), (char **)opts.data()), + Catch::Matchers::Contains("Missing input")); } TEST_CASE("Incompatible arguments", "[NoDB]") diff -Nru osm2pgsql-1.8.1+ds/tests/test-options-projection.cpp osm2pgsql-1.9.0+ds/tests/test-options-projection.cpp --- osm2pgsql-1.8.1+ds/tests/test-options-projection.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-options-projection.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -12,6 +12,8 @@ #include #include "common-import.hpp" + +#include "command-line-parser.hpp" #include "reprojection.hpp" static testing::db::import_t db; @@ -73,8 +75,8 @@ option_params.push_back("foo"); - options_t const options{(int)option_params.size(), - (char **)option_params.data()}; + auto const options = parse_command_line((int)option_params.size(), + (char **)option_params.data()); if (!proj_name.empty()) { CHECK(options.projection->target_desc() == proj_name); diff -Nru osm2pgsql-1.8.1+ds/tests/test-osm-file-parsing.cpp osm2pgsql-1.9.0+ds/tests/test-osm-file-parsing.cpp --- osm2pgsql-1.8.1+ds/tests/test-osm-file-parsing.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-osm-file-parsing.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,309 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include + +#include "middle.hpp" +#include "output-null.hpp" + +#include "common-import.hpp" +#include "common-options.hpp" + +struct type_stats_t +{ + unsigned added = 0; + unsigned modified = 0; + unsigned deleted = 0; +}; + +struct counting_middle_t : public middle_t +{ + explicit counting_middle_t(bool append) + : middle_t(nullptr), m_append(append) + {} + + void start() override + { + assert(m_middle_state == middle_state::constructed); +#ifndef NDEBUG + m_middle_state = middle_state::node; +#endif + } + + void stop() override { assert(m_middle_state == middle_state::done); } + + void cleanup() {} + + void node(osmium::Node const &node) override { + assert(m_middle_state == middle_state::node); + + if (m_append) { + ++node_count.deleted; + if (!node.deleted()) { + ++node_count.added; + } + return; + } + ++node_count.added; + } + + void way(osmium::Way const &way) override { + assert(m_middle_state == middle_state::way); + + if (m_append) { + ++way_count.deleted; + if (!way.deleted()) { + ++way_count.added; + } + return; + } + ++way_count.added; + } + + void relation(osmium::Relation const &relation) override { + assert(m_middle_state == middle_state::relation); + + if (m_append) { + ++relation_count.deleted; + if (!relation.deleted()) { + ++relation_count.added; + } + return; + } + ++relation_count.added; + } + + std::shared_ptr get_query_instance() override + { + return std::shared_ptr{}; + } + + type_stats_t node_count, way_count, relation_count; + bool m_append; +}; + +struct counting_output_t : public output_null_t +{ + explicit counting_output_t(options_t const &options) + : output_null_t(nullptr, nullptr, options) + {} + + std::shared_ptr + clone(std::shared_ptr const &, + std::shared_ptr const &) const override + { + return std::make_shared(*get_options()); + } + + void node_add(osmium::Node const &n) override + { + ++node.added; + sum_ids += n.id(); + } + + void way_add(osmium::Way *w) override + { + ++way.added; + sum_ids += w->id(); + sum_nds += w->nodes().size(); + } + + void relation_add(osmium::Relation const &r) override + { + ++relation.added; + sum_ids += r.id(); + sum_members += r.members().size(); + } + + void node_modify(osmium::Node const &) override { ++node.modified; } + + void way_modify(osmium::Way *) override { ++way.modified; } + + void relation_modify(osmium::Relation const &) override + { + ++relation.modified; + } + + void node_delete(osmid_t) override { ++node.deleted; } + + void way_delete(osmid_t) override { ++way.deleted; } + + void relation_delete(osmid_t) override { ++relation.deleted; } + + type_stats_t node, way, relation; + uint64_t sum_ids = 0; + std::size_t sum_nds = 0; + std::size_t sum_members = 0; +}; + +struct counts_t { + std::size_t nodes_changed = 0; + std::size_t ways_changed = 0; +}; + +/** + * This pseudo-dependency manager is just used for testing. It counts how + * often the *_changed() member functions are called. + */ +class counting_dependency_manager_t : public dependency_manager_t +{ +public: + explicit counting_dependency_manager_t(std::shared_ptr counts) + : m_counts(std::move(counts)) + {} + + void node_changed(osmid_t) override { ++m_counts->nodes_changed; } + void way_changed(osmid_t) override { ++m_counts->ways_changed; } + +private: + std::shared_ptr m_counts; +}; + +TEST_CASE("parse xml file") +{ + options_t const options = testing::opt_t().slim(); + + auto const middle = std::make_shared(false); + middle->start(); + + auto const output = std::make_shared(options); + + auto counts = std::make_shared(); + auto dependency_manager = + std::make_unique(counts); + + testing::parse_file(options, std::move(dependency_manager), middle, + output, "test_multipolygon.osm", false); + + REQUIRE(output->sum_ids == 4728); + REQUIRE(output->sum_nds == 186); + REQUIRE(output->sum_members == 146); + REQUIRE(output->node.added == 0); + REQUIRE(output->node.modified == 0); + REQUIRE(output->node.deleted == 0); + REQUIRE(output->way.added == 48); + REQUIRE(output->way.modified == 0); + REQUIRE(output->way.deleted == 0); + REQUIRE(output->relation.added == 40); + REQUIRE(output->relation.modified == 0); + REQUIRE(output->relation.deleted == 0); + + auto const *mid_test = middle.get(); + REQUIRE(mid_test->node_count.added == 353); + REQUIRE(mid_test->node_count.deleted == 0); + REQUIRE(mid_test->way_count.added == 140); + REQUIRE(mid_test->way_count.deleted == 0); + REQUIRE(mid_test->relation_count.added == 40); + REQUIRE(mid_test->relation_count.deleted == 0); + + REQUIRE(counts->nodes_changed == 0); + REQUIRE(counts->ways_changed == 0); +} + +TEST_CASE("parse diff file") +{ + options_t const options = testing::opt_t().slim().append(); + + auto const middle = std::make_shared(true); + middle->start(); + + auto const output = std::make_shared(options); + + auto counts = std::make_shared(); + auto dependency_manager = + std::make_unique(counts); + + testing::parse_file(options, std::move(dependency_manager), middle, + output, "008-ch.osc.gz", false); + + REQUIRE(output->node.added == 0); + REQUIRE(output->node.modified == 153); + REQUIRE(output->node.deleted == 17796); + REQUIRE(output->way.added == 0); + REQUIRE(output->way.modified == 161); + REQUIRE(output->way.deleted == 4); + REQUIRE(output->relation.added == 0); + REQUIRE(output->relation.modified == 11); + REQUIRE(output->relation.deleted == 1); + + auto *mid_test = middle.get(); + REQUIRE(mid_test->node_count.added == 1176); + REQUIRE(mid_test->node_count.deleted == 17949); + REQUIRE(mid_test->way_count.added == 161); + REQUIRE(mid_test->way_count.deleted == 165); + REQUIRE(mid_test->relation_count.added == 11); + REQUIRE(mid_test->relation_count.deleted == 12); + + REQUIRE(counts->nodes_changed == 1176); + REQUIRE(counts->ways_changed == 161); +} + +TEST_CASE("parse xml file with extra args") +{ + options_t options = testing::opt_t().slim().srs(PROJ_SPHERE_MERC); + options.extra_attributes = true; + + auto const middle = std::make_shared(false); + middle->start(); + + auto const output = std::make_shared(options); + + auto counts = std::make_shared(); + auto dependency_manager = + std::make_unique(counts); + + testing::parse_file(options, std::move(dependency_manager), middle, + output, "test_multipolygon.osm", false); + + REQUIRE(output->sum_ids == 73514); + REQUIRE(output->sum_nds == 495); + REQUIRE(output->sum_members == 146); + REQUIRE(output->node.added == 353); + REQUIRE(output->node.modified == 0); + REQUIRE(output->node.deleted == 0); + REQUIRE(output->way.added == 140); + REQUIRE(output->way.modified == 0); + REQUIRE(output->way.deleted == 0); + REQUIRE(output->relation.added == 40); + REQUIRE(output->relation.modified == 0); + REQUIRE(output->relation.deleted == 0); + + auto const *mid_test = middle.get(); + REQUIRE(mid_test->node_count.added == 353); + REQUIRE(mid_test->node_count.deleted == 0); + REQUIRE(mid_test->way_count.added == 140); + REQUIRE(mid_test->way_count.deleted == 0); + REQUIRE(mid_test->relation_count.added == 40); + REQUIRE(mid_test->relation_count.deleted == 0); + + REQUIRE(counts->nodes_changed == 0); + REQUIRE(counts->ways_changed == 0); +} + +TEST_CASE("invalid location") +{ + options_t const options = testing::opt_t(); + + auto const middle = std::make_shared(false); + middle->start(); + + auto const output = std::make_shared(options); + + auto counts = std::make_shared(); + auto dependency_manager = + std::make_unique(counts); + + testing::parse_file(options, std::move(dependency_manager), middle, + output, "test_invalid_location.osm", false); + + REQUIRE(output->node.added == 0); + REQUIRE(output->way.added == 0); + REQUIRE(output->relation.added == 0); +} + diff -Nru osm2pgsql-1.8.1+ds/tests/test-output-flex-schema.cpp osm2pgsql-1.9.0+ds/tests/test-output-flex-schema.cpp --- osm2pgsql-1.8.1+ds/tests/test-output-flex-schema.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-output-flex-schema.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -27,17 +27,19 @@ REQUIRE_NOTHROW(db.run_file(options, data_file)); - REQUIRE(1 == conn.get_count("pg_namespace", "nspname = 'myschema'")); - REQUIRE(1 == conn.get_count("pg_tables", "schemaname = 'myschema'")); + REQUIRE(1 == + conn.get_count("pg_catalog.pg_namespace", "nspname = 'myschema'")); + REQUIRE(1 == + conn.get_count("pg_catalog.pg_tables", "schemaname = 'myschema'")); REQUIRE(7103 == conn.get_count("myschema.osm2pgsql_test_line")); REQUIRE(1 == - conn.get_count("pg_proc", + conn.get_count("pg_catalog.pg_proc", "proname = 'osm2pgsql_test_line_osm2pgsql_valid'")); - REQUIRE(1 == conn.get_count("pg_trigger")); + REQUIRE(1 == conn.get_count("pg_catalog.pg_trigger")); REQUIRE(1 == - conn.get_count("pg_trigger", + conn.get_count("pg_catalog.pg_trigger", "tgname = 'osm2pgsql_test_line_osm2pgsql_valid'")); } diff -Nru osm2pgsql-1.8.1+ds/tests/test-output-flex-tablespace.cpp osm2pgsql-1.9.0+ds/tests/test-output-flex-tablespace.cpp --- osm2pgsql-1.8.1+ds/tests/test-output-flex-tablespace.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-output-flex-tablespace.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -20,8 +20,8 @@ { { auto conn = db.db().connect(); - REQUIRE(1 == - conn.get_count("pg_tablespace", "spcname = 'tablespacetest'")); + REQUIRE(1 == conn.get_count("pg_catalog.pg_tablespace", + "spcname = 'tablespacetest'")); } options_t options = testing::opt_t().slim().flex(conf_file); diff -Nru osm2pgsql-1.8.1+ds/tests/test-output-pgsql-tablespace.cpp osm2pgsql-1.9.0+ds/tests/test-output-pgsql-tablespace.cpp --- osm2pgsql-1.8.1+ds/tests/test-output-pgsql-tablespace.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-output-pgsql-tablespace.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -26,8 +26,8 @@ { { auto conn = db.db().connect(); - REQUIRE(1 == - conn.get_count("pg_tablespace", "spcname = 'tablespacetest'")); + REQUIRE(1 == conn.get_count("pg_catalog.pg_tablespace", + "spcname = 'tablespacetest'")); } options_t options = testing::opt_t().slim(); diff -Nru osm2pgsql-1.8.1+ds/tests/test-parse-osmium.cpp osm2pgsql-1.9.0+ds/tests/test-parse-osmium.cpp --- osm2pgsql-1.8.1+ds/tests/test-parse-osmium.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-parse-osmium.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,287 +0,0 @@ -/** - * SPDX-License-Identifier: GPL-2.0-or-later - * - * This file is part of osm2pgsql (https://osm2pgsql.org/). - * - * Copyright (C) 2006-2023 by the osm2pgsql developer community. - * For a full list of authors see the git log. - */ - -#include - -#include "middle.hpp" -#include "output-null.hpp" - -#include "common-import.hpp" -#include "common-options.hpp" - -struct type_stats_t -{ - unsigned added = 0; - unsigned modified = 0; - unsigned deleted = 0; -}; - -struct counting_middle_t : public middle_t -{ - explicit counting_middle_t(bool append) - : middle_t(nullptr), m_append(append) - {} - - void start() override {} - void stop() override {} - void cleanup() {} - - void node(osmium::Node const &node) override { - if (m_append) { - ++node_count.deleted; - if (!node.deleted()) { - ++node_count.added; - } - return; - } - ++node_count.added; - } - - void way(osmium::Way const &way) override { - if (m_append) { - ++way_count.deleted; - if (!way.deleted()) { - ++way_count.added; - } - return; - } - ++way_count.added; - } - - void relation(osmium::Relation const &relation) override { - if (m_append) { - ++relation_count.deleted; - if (!relation.deleted()) { - ++relation_count.added; - } - return; - } - ++relation_count.added; - } - - std::shared_ptr get_query_instance() override - { - return std::shared_ptr{}; - } - - type_stats_t node_count, way_count, relation_count; - bool m_append; -}; - -struct counting_output_t : public output_null_t -{ - explicit counting_output_t(options_t const &options) - : output_null_t(nullptr, nullptr, options) - {} - - std::shared_ptr - clone(std::shared_ptr const &, - std::shared_ptr const &) const override - { - return std::make_shared(*get_options()); - } - - void node_add(osmium::Node const &n) override - { - ++node.added; - sum_ids += n.id(); - } - - void way_add(osmium::Way *w) override - { - ++way.added; - sum_ids += w->id(); - sum_nds += w->nodes().size(); - } - - void relation_add(osmium::Relation const &r) override - { - ++relation.added; - sum_ids += r.id(); - sum_members += r.members().size(); - } - - void node_modify(osmium::Node const &) override { ++node.modified; } - - void way_modify(osmium::Way *) override { ++way.modified; } - - void relation_modify(osmium::Relation const &) override - { - ++relation.modified; - } - - void node_delete(osmid_t) override { ++node.deleted; } - - void way_delete(osmid_t) override { ++way.deleted; } - - void relation_delete(osmid_t) override { ++relation.deleted; } - - type_stats_t node, way, relation; - uint64_t sum_ids = 0; - std::size_t sum_nds = 0; - std::size_t sum_members = 0; -}; - -struct counts_t { - std::size_t nodes_changed = 0; - std::size_t ways_changed = 0; -}; - -/** - * This pseudo-dependency manager is just used for testing. It counts how - * often the *_changed() member functions are called. - */ -class counting_dependency_manager_t : public dependency_manager_t -{ -public: - explicit counting_dependency_manager_t(std::shared_ptr counts) - : m_counts(std::move(counts)) - {} - - void node_changed(osmid_t) override { ++m_counts->nodes_changed; } - void way_changed(osmid_t) override { ++m_counts->ways_changed; } - -private: - std::shared_ptr m_counts; -}; - -TEST_CASE("parse xml file") -{ - options_t const options = testing::opt_t().slim(); - - auto const middle = std::make_shared(false); - auto const output = std::make_shared(options); - - auto counts = std::make_shared(); - auto dependency_manager = - std::make_unique(counts); - - testing::parse_file(options, std::move(dependency_manager), middle, - output, "test_multipolygon.osm", false); - - REQUIRE(output->sum_ids == 4728); - REQUIRE(output->sum_nds == 186); - REQUIRE(output->sum_members == 146); - REQUIRE(output->node.added == 0); - REQUIRE(output->node.modified == 0); - REQUIRE(output->node.deleted == 0); - REQUIRE(output->way.added == 48); - REQUIRE(output->way.modified == 0); - REQUIRE(output->way.deleted == 0); - REQUIRE(output->relation.added == 40); - REQUIRE(output->relation.modified == 0); - REQUIRE(output->relation.deleted == 0); - - auto const *mid_test = middle.get(); - REQUIRE(mid_test->node_count.added == 353); - REQUIRE(mid_test->node_count.deleted == 0); - REQUIRE(mid_test->way_count.added == 140); - REQUIRE(mid_test->way_count.deleted == 0); - REQUIRE(mid_test->relation_count.added == 40); - REQUIRE(mid_test->relation_count.deleted == 0); - - REQUIRE(counts->nodes_changed == 0); - REQUIRE(counts->ways_changed == 0); -} - -TEST_CASE("parse diff file") -{ - options_t const options = testing::opt_t().slim().append(); - - auto const middle = std::make_shared(true); - auto const output = std::make_shared(options); - - auto counts = std::make_shared(); - auto dependency_manager = - std::make_unique(counts); - - testing::parse_file(options, std::move(dependency_manager), middle, - output, "008-ch.osc.gz", false); - - REQUIRE(output->node.added == 0); - REQUIRE(output->node.modified == 153); - REQUIRE(output->node.deleted == 17796); - REQUIRE(output->way.added == 0); - REQUIRE(output->way.modified == 161); - REQUIRE(output->way.deleted == 4); - REQUIRE(output->relation.added == 0); - REQUIRE(output->relation.modified == 11); - REQUIRE(output->relation.deleted == 1); - - auto *mid_test = middle.get(); - REQUIRE(mid_test->node_count.added == 1176); - REQUIRE(mid_test->node_count.deleted == 17949); - REQUIRE(mid_test->way_count.added == 161); - REQUIRE(mid_test->way_count.deleted == 165); - REQUIRE(mid_test->relation_count.added == 11); - REQUIRE(mid_test->relation_count.deleted == 12); - - REQUIRE(counts->nodes_changed == 1176); - REQUIRE(counts->ways_changed == 161); -} - -TEST_CASE("parse xml file with extra args") -{ - options_t options = testing::opt_t().slim().srs(PROJ_SPHERE_MERC); - options.extra_attributes = true; - - auto const middle = std::make_shared(false); - auto const output = std::make_shared(options); - - auto counts = std::make_shared(); - auto dependency_manager = - std::make_unique(counts); - - testing::parse_file(options, std::move(dependency_manager), middle, - output, "test_multipolygon.osm", false); - - REQUIRE(output->sum_ids == 73514); - REQUIRE(output->sum_nds == 495); - REQUIRE(output->sum_members == 146); - REQUIRE(output->node.added == 353); - REQUIRE(output->node.modified == 0); - REQUIRE(output->node.deleted == 0); - REQUIRE(output->way.added == 140); - REQUIRE(output->way.modified == 0); - REQUIRE(output->way.deleted == 0); - REQUIRE(output->relation.added == 40); - REQUIRE(output->relation.modified == 0); - REQUIRE(output->relation.deleted == 0); - - auto const *mid_test = middle.get(); - REQUIRE(mid_test->node_count.added == 353); - REQUIRE(mid_test->node_count.deleted == 0); - REQUIRE(mid_test->way_count.added == 140); - REQUIRE(mid_test->way_count.deleted == 0); - REQUIRE(mid_test->relation_count.added == 40); - REQUIRE(mid_test->relation_count.deleted == 0); - - REQUIRE(counts->nodes_changed == 0); - REQUIRE(counts->ways_changed == 0); -} - -TEST_CASE("invalid location") -{ - options_t const options = testing::opt_t(); - - auto const middle = std::make_shared(false); - auto const output = std::make_shared(options); - - auto counts = std::make_shared(); - auto dependency_manager = - std::make_unique(counts); - - testing::parse_file(options, std::move(dependency_manager), middle, - output, "test_invalid_location.osm", false); - - REQUIRE(output->node.added == 0); - REQUIRE(output->way.added == 0); - REQUIRE(output->relation.added == 0); -} - diff -Nru osm2pgsql-1.8.1+ds/tests/test-pgsql-capabilities.cpp osm2pgsql-1.9.0+ds/tests/test-pgsql-capabilities.cpp --- osm2pgsql-1.8.1+ds/tests/test-pgsql-capabilities.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-pgsql-capabilities.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -45,6 +45,14 @@ REQUIRE_FALSE(has_index_method("xzxzxzxz")); } +TEST_CASE("has_table() should work") +{ + init_database_capabilities(db.db().connect()); + REQUIRE_FALSE(has_table("public", "foo")); + REQUIRE_FALSE(has_table("someschema", "foo")); + REQUIRE(has_table("public", "spatial_ref_sys")); +} + TEST_CASE("PostgreSQL version") { init_database_capabilities(db.db().connect()); diff -Nru osm2pgsql-1.8.1+ds/tests/test-pgsql.cpp osm2pgsql-1.9.0+ds/tests/test-pgsql.cpp --- osm2pgsql-1.8.1+ds/tests/test-pgsql.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-pgsql.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -24,9 +24,9 @@ REQUIRE(tablespace_clause("foo") == R"( TABLESPACE "foo")"); } -TEST_CASE("Table name without schema") +TEST_CASE("Table name with public schema") { - REQUIRE(qualified_name("", "foo") == R"("foo")"); + REQUIRE(qualified_name("public", "foo") == R"("public"."foo")"); } TEST_CASE("Table name with schema") @@ -36,7 +36,7 @@ TEST_CASE("query with SELECT should work") { - auto conn = db.db().connect(); + auto const conn = db.db().connect(); auto const result = conn.exec("SELECT 42"); REQUIRE(result.status() == PGRES_TUPLES_OK); REQUIRE(result.num_fields() == 1); @@ -46,31 +46,19 @@ TEST_CASE("query with invalid SQL should fail") { - auto conn = db.db().connect(); + auto const conn = db.db().connect(); REQUIRE_THROWS(conn.exec("NOT-VALID-SQL")); } TEST_CASE("exec with invalid SQL should fail") { - auto conn = db.db().connect(); + auto const conn = db.db().connect(); REQUIRE_THROWS(conn.exec("XYZ")); } -TEST_CASE("exec_prepared without parameters should work") -{ - auto conn = db.db().connect(); - conn.exec("PREPARE test AS SELECT 42"); - - auto const result = conn.exec_prepared("test"); - REQUIRE(result.status() == PGRES_TUPLES_OK); - REQUIRE(result.num_fields() == 1); - REQUIRE(result.num_tuples() == 1); - REQUIRE(result.get(0, 0) == "42"); -} - TEST_CASE("exec_prepared with single string parameters should work") { - auto conn = db.db().connect(); + auto const conn = db.db().connect(); conn.exec("PREPARE test(int) AS SELECT $1"); auto const result = conn.exec_prepared("test", "17"); @@ -82,7 +70,7 @@ TEST_CASE("exec_prepared with string parameters should work") { - auto conn = db.db().connect(); + auto const conn = db.db().connect(); conn.exec("PREPARE test(int, int, int, int, int)" " AS SELECT $1 + $2 + $3 + $4 + $5"); @@ -98,7 +86,7 @@ TEST_CASE("exec_prepared with non-string parameters should work") { - auto conn = db.db().connect(); + auto const conn = db.db().connect(); conn.exec("PREPARE test(int, int, int) AS SELECT $1 + $2 + $3"); auto const result = conn.exec_prepared("test", 1, 2.0, 3ULL); @@ -107,3 +95,43 @@ REQUIRE(result.num_tuples() == 1); REQUIRE(result.get(0, 0) == "6"); } + +TEST_CASE("exec_prepared with binary parameter should work") +{ + auto const conn = db.db().connect(); + conn.exec("PREPARE test(bytea) AS SELECT length($1)"); + + binary_param const p{"foo \x01 bar"}; + auto const result = conn.exec_prepared("test", p); + REQUIRE(result.status() == PGRES_TUPLES_OK); + REQUIRE(result.num_fields() == 1); + REQUIRE(result.num_tuples() == 1); + REQUIRE(result.get(0, 0) == "9"); +} + +TEST_CASE("exec_prepared with mixed parameter types should work") +{ + auto const conn = db.db().connect(); + conn.exec("PREPARE test(text, bytea, int) AS" + " SELECT length($1) + length($2) + $3"); + + std::string const p1{"foo bar"}; + binary_param const p2{"foo \x01 bar"}; + int const p3 = 17; + auto const result = conn.exec_prepared("test", p1, p2, p3); + REQUIRE(result.status() == PGRES_TUPLES_OK); + REQUIRE(result.num_fields() == 1); + REQUIRE(result.num_tuples() == 1); + REQUIRE(result.get(0, 0) == "33"); // 7 + 9 + 17 +} + +TEST_CASE("create table and insert something") +{ + auto const conn = db.db().connect(); + conn.exec("CREATE TABLE foo (x int)"); + auto const result = conn.exec("INSERT INTO foo (x) VALUES (1), (2)"); + REQUIRE(result.status() == PGRES_COMMAND_OK); + REQUIRE(result.num_fields() == 0); + REQUIRE(result.num_tuples() == 0); + REQUIRE(result.affected_rows() == 2); +} diff -Nru osm2pgsql-1.8.1+ds/tests/test-properties.cpp osm2pgsql-1.9.0+ds/tests/test-properties.cpp --- osm2pgsql-1.8.1+ds/tests/test-properties.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-properties.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -0,0 +1,155 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include + +#include "properties.hpp" + +#include "common-pg.hpp" + +TEST_CASE("Store and retrieve properties (memory only)") +{ + properties_t properties{"", "public"}; + + properties.set_string("foo", "firstvalue"); + properties.set_string("foo", "bar"); // overwriting is okay + properties.set_string("number", "astring"); + properties.set_int("number", 123); // overwriting with other type okay + properties.set_bool("decide", true); + properties.set_string("empty", ""); // empty string is okay + + REQUIRE(properties.get_string("foo", "baz") == "bar"); + REQUIRE(properties.get_string("something", "baz") == "baz"); + REQUIRE(properties.get_string("empty", "baz").empty()); + REQUIRE_THROWS(properties.get_int("foo", 1)); + REQUIRE_THROWS(properties.get_bool("foo", true)); + + REQUIRE(properties.get_int("number", 42) == 123); + REQUIRE(properties.get_int("anumber", 42) == 42); + REQUIRE(properties.get_string("number", "x") == "123"); + REQUIRE_THROWS(properties.get_bool("number", true)); + + REQUIRE(properties.get_bool("decide", false)); + REQUIRE(properties.get_bool("unknown", true)); + REQUIRE_FALSE(properties.get_bool("unknown", false)); + REQUIRE(properties.get_string("decide", "x") == "true"); + REQUIRE_THROWS(properties.get_int("decide", 123)); +} + +TEST_CASE("Store and retrieve properties (with database)") +{ + for (std::string const schema : {"public", "middleschema"}) { + testing::pg::tempdb_t const db; + auto conn = db.connect(); + if (schema != "public") { + conn.exec("CREATE SCHEMA IF NOT EXISTS {};", schema); + } + + { + properties_t properties{db.conninfo(), schema}; + + properties.set_string("foo", "bar"); + properties.set_string("empty", ""); + properties.set_int("number", 123); + properties.set_bool("decide", true); + + properties.store(); + } + + { + init_database_capabilities(conn); + std::string const full_table_name = + (schema.empty() ? "" : schema + ".") + "osm2pgsql_properties"; + + REQUIRE(conn.get_count(full_table_name.c_str()) == 4); + REQUIRE(conn.get_count(full_table_name.c_str(), + "property='foo' AND value='bar'") == 1); + REQUIRE(conn.get_count(full_table_name.c_str(), + "property='empty' AND value=''") == 1); + REQUIRE(conn.get_count(full_table_name.c_str(), + "property='number' AND value='123'") == 1); + REQUIRE(conn.get_count(full_table_name.c_str(), + "property='decide' AND value='true'") == 1); + + properties_t properties{db.conninfo(), schema}; + REQUIRE(properties.load()); + + REQUIRE(properties.get_string("foo", "baz") == "bar"); + REQUIRE(properties.get_string("something", "baz") == "baz"); + REQUIRE(properties.get_string("empty", "baz").empty()); + REQUIRE_THROWS(properties.get_int("foo", 1)); + REQUIRE_THROWS(properties.get_bool("foo", true)); + + REQUIRE(properties.get_int("number", 42) == 123); + REQUIRE(properties.get_int("anumber", 42) == 42); + REQUIRE(properties.get_string("number", "x") == "123"); + REQUIRE_THROWS(properties.get_bool("number", true)); + + REQUIRE(properties.get_bool("decide", false)); + REQUIRE(properties.get_bool("unknown", true)); + REQUIRE_FALSE(properties.get_bool("unknown", false)); + REQUIRE(properties.get_string("decide", "x") == "true"); + REQUIRE_THROWS(properties.get_int("decide", 123)); + } + } +} + +TEST_CASE("Update existing properties in database") +{ + testing::pg::tempdb_t const db; + auto conn = db.connect(); + + { + properties_t properties{db.conninfo(), "public"}; + + properties.set_string("a", "xxx"); + properties.set_string("b", "yyy"); + + properties.store(); + } + + { + init_database_capabilities(conn); + REQUIRE(conn.get_count("osm2pgsql_properties") == 2); + + properties_t properties{db.conninfo(), "public"}; + REQUIRE(properties.load()); + + REQUIRE(properties.get_string("a", "def") == "xxx"); + REQUIRE(properties.get_string("b", "def") == "yyy"); + + properties.set_string("a", "zzz", false); + properties.set_string("b", "zzz", true); + + // both are updated in memory + REQUIRE(properties.get_string("a", "def") == "zzz"); + REQUIRE(properties.get_string("b", "def") == "zzz"); + } + + { + REQUIRE(conn.get_count("osm2pgsql_properties") == 2); + + properties_t properties{db.conninfo(), "public"}; + REQUIRE(properties.load()); + + // only "b" was updated in the database + REQUIRE(properties.get_string("a", "def") == "xxx"); + REQUIRE(properties.get_string("b", "def") == "zzz"); + } +} + +TEST_CASE("Load returns false if there are no properties in database") +{ + testing::pg::tempdb_t const db; + auto conn = db.connect(); + init_database_capabilities(conn); + + properties_t properties{db.conninfo(), "public"}; + REQUIRE_FALSE(properties.load()); +} diff -Nru osm2pgsql-1.8.1+ds/tests/test-tile.cpp osm2pgsql-1.9.0+ds/tests/test-tile.cpp --- osm2pgsql-1.8.1+ds/tests/test-tile.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-tile.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -59,6 +59,21 @@ REQUIRE(tile.ymin() == Approx(-tile_t::half_earth_circumference)); REQUIRE(tile.xmax() == Approx(tile_t::half_earth_circumference)); REQUIRE(tile.ymax() == Approx(tile_t::half_earth_circumference)); + REQUIRE(tile.box().min_x() == Approx(-tile_t::half_earth_circumference)); + REQUIRE(tile.box().min_y() == Approx(-tile_t::half_earth_circumference)); + REQUIRE(tile.box().max_x() == Approx(tile_t::half_earth_circumference)); + REQUIRE(tile.box().max_y() == Approx(tile_t::half_earth_circumference)); + + // Bounding box with margin will not get larger, because it is always + // clamped to the full extent of the map. + REQUIRE(tile.xmin(0.1) == Approx(-tile_t::half_earth_circumference)); + REQUIRE(tile.ymin(0.1) == Approx(-tile_t::half_earth_circumference)); + REQUIRE(tile.xmax(0.1) == Approx(tile_t::half_earth_circumference)); + REQUIRE(tile.ymax(0.1) == Approx(tile_t::half_earth_circumference)); + REQUIRE(tile.box(0.1).min_x() == Approx(-tile_t::half_earth_circumference)); + REQUIRE(tile.box(0.1).min_y() == Approx(-tile_t::half_earth_circumference)); + REQUIRE(tile.box(0.1).max_x() == Approx(tile_t::half_earth_circumference)); + REQUIRE(tile.box(0.1).max_y() == Approx(tile_t::half_earth_circumference)); REQUIRE(tile.center().x() == Approx(0.0)); REQUIRE(tile.center().y() == Approx(0.0)); @@ -79,10 +94,28 @@ { tile_t const tile{2, 1, 2}; - REQUIRE(tile.xmin() == Approx(-tile_t::half_earth_circumference / 2)); - REQUIRE(tile.ymin() == Approx(-tile_t::half_earth_circumference / 2)); - REQUIRE(tile.xmax() == Approx(0.0)); - REQUIRE(tile.ymax() == Approx(0.0)); + double min = -tile_t::half_earth_circumference / 2; + double max = 0.0; + REQUIRE(tile.xmin() == Approx(min)); + REQUIRE(tile.ymin() == Approx(min)); + REQUIRE(tile.xmax() == Approx(max)); + REQUIRE(tile.ymax() == Approx(max)); + CHECK(tile.box().min_x() == tile.xmin()); + CHECK(tile.box().min_y() == tile.ymin()); + CHECK(tile.box().max_x() == tile.xmax()); + CHECK(tile.box().max_y() == tile.ymax()); + + // Bounding box of tile with 50% margin on all sides. + min -= tile_t::half_earth_circumference / 4; + max += tile_t::half_earth_circumference / 4; + CHECK(tile.xmin(0.5) == Approx(min)); + CHECK(tile.ymin(0.5) == Approx(min)); + CHECK(tile.xmax(0.5) == Approx(max)); + CHECK(tile.ymax(0.5) == Approx(max)); + CHECK(tile.box(0.5).min_x() == tile.xmin(0.5)); + CHECK(tile.box(0.5).min_y() == tile.ymin(0.5)); + CHECK(tile.box(0.5).max_x() == tile.xmax(0.5)); + CHECK(tile.box(0.5).max_y() == tile.ymax(0.5)); REQUIRE(tile.center().x() == Approx(-tile_t::half_earth_circumference / 4)); REQUIRE(tile.center().y() == Approx(-tile_t::half_earth_circumference / 4)); diff -Nru osm2pgsql-1.8.1+ds/tests/test-util.cpp osm2pgsql-1.9.0+ds/tests/test-util.cpp --- osm2pgsql-1.8.1+ds/tests/test-util.cpp 2023-02-13 09:02:52.000000000 +0000 +++ osm2pgsql-1.9.0+ds/tests/test-util.cpp 2023-08-15 19:18:18.000000000 +0000 @@ -65,7 +65,7 @@ t.emplace_back("baz"); REQUIRE(util::find_by_name(t, "") == nullptr); - REQUIRE(util::find_by_name(t, "foo") == &t[0]); + REQUIRE(util::find_by_name(t, "foo") == t.data()); REQUIRE(util::find_by_name(t, "bar") == &t[1]); REQUIRE(util::find_by_name(t, "baz") == &t[2]); REQUIRE(util::find_by_name(t, "nothing") == nullptr);