diff -Nru osm2pgsql-0.82.0/AUTHORS osm2pgsql-0.88.1/AUTHORS --- osm2pgsql-0.82.0/AUTHORS 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/AUTHORS 2015-08-15 05:28:11.000000000 +0000 @@ -1,3 +1,3 @@ osm2pgsql was written by Jon Burgess, Artem Pavlenko, Martijn van Oosterhout -Sarah Hoffman, Kai Krueger, Frederik Ramm, Brian Quinion and other -OpenStreetMap project members. \ No newline at end of file +Sarah Hoffmann, Kai Krueger, Frederik Ramm, Brian Quinion, Matt Amos, +Kevin Kreiser, Paul Norman and other OpenStreetMap project members. diff -Nru osm2pgsql-0.82.0/binarysearcharray.c osm2pgsql-0.88.1/binarysearcharray.c --- osm2pgsql-0.82.0/binarysearcharray.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/binarysearcharray.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,118 +0,0 @@ -#include -#include -#include -#include - -#include "osmtypes.h" -#include "binarysearcharray.h" - -static int binary_search_lookup(struct binary_search_array * array, int key) -{ - int a = 0; - int b = array->size - 1; - while (a <= b) - { - int pivot = ((b - a) >> 1) + a; - if (array->array[pivot].key == key) - { - return pivot; - } - else if (array->array[pivot].key > key) - { - b = pivot - 1; - } - else - { - a = pivot + 1; - } - } - if ((a < array->size) && (array->array[a].key < key)) - a++; - return a | (1 << (sizeof(int) * 8 - 1)); -} - -osmid_t binary_search_get(struct binary_search_array * array, int key) -{ - int idx; - if (array->size == 0) - return -1; - idx = binary_search_lookup(array, key); - if (idx < 0) - { - return -1; - } - else - { - return array->array[idx].value; - } - exit(1); -} - -void binary_search_remove(struct binary_search_array * array, int key) -{ - int idx = binary_search_lookup(array, key); - if (idx < 0) - { - return; - } - else - { - memmove(&(array->array[idx]), &(array->array[idx + 1]), - sizeof(struct key_val_tuple) * (array->capacity - idx - 1)); - array->size--; - } -} - -void binary_search_add(struct binary_search_array * array, int key, - osmid_t value) -{ - int idx; - if (array->size < array->capacity) - { - if (array->size == 0) - { - array->array[0].key = key; - array->array[0].value = value; - array->size++; - return; - } - idx = binary_search_lookup(array, key); - if (idx < 0) - { - idx = idx & (~(1 << (sizeof(int) * 8 - 1))); - memmove(&(array->array[idx + 1]), &(array->array[idx]), - sizeof(struct key_val_tuple) * (array->capacity - idx - 1)); - array->array[idx].key = key; - array->array[idx].value = value; - array->size++; - } - else - { - fprintf(stderr, "dupplicate!\n"); - exit(1); - } - } -} - -struct binary_search_array * init_search_array(int capacity) -{ - struct binary_search_array * array = calloc(1, - sizeof(struct binary_search_array)); - array->array = calloc(capacity + 1, sizeof(struct key_val_tuple)); - if (!array->array) { - fprintf(stderr, "Out of memory trying to allocate %li bytes for binary search array\n", ((capacity + 1) * sizeof(struct key_val_tuple))); - exit_nicely(); - } - array->capacity = capacity; - array->size = 0; - return array; -} - -void shutdown_search_array(struct binary_search_array ** array) -{ - free((*array)->array); - (*array)->array = NULL; - (*array)->capacity = 0; - free(*array); - *array = NULL; -} diff -Nru osm2pgsql-0.82.0/binarysearcharray.h osm2pgsql-0.88.1/binarysearcharray.h --- osm2pgsql-0.82.0/binarysearcharray.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/binarysearcharray.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,20 +0,0 @@ - - - -struct key_val_tuple { - int key; - osmid_t value; -}; - -struct binary_search_array { - int capacity; - int size; - struct key_val_tuple * array; -}; - -void binary_search_remove(struct binary_search_array * array, int key); -void binary_search_add(struct binary_search_array * array, int key, osmid_t value); -osmid_t binary_search_get(struct binary_search_array * array, int key); -struct binary_search_array * init_search_array(int capacity); -void shutdown_search_array(struct binary_search_array ** array); - diff -Nru osm2pgsql-0.82.0/build_geometry.cpp osm2pgsql-0.88.1/build_geometry.cpp --- osm2pgsql-0.82.0/build_geometry.cpp 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/build_geometry.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,550 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#include -#include -#include -#include - -/* Need to know which geos version we have to work out which headers to include */ -#include - -/* geos (3.0.0+) */ -#if (GEOS_VERSION_MAJOR==3) -#if (GEOS_VERSION_MINOR>=1) -/* Prepared geometries are new in 3.1.0 */ -#define HAS_PREPARED_GEOMETRIES -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace geos::geom; -using namespace geos::io; -using namespace geos::util; -using namespace geos::operation::linemerge; -#else -/* geos-2.2.3 */ -#include -#include -#include -using namespace geos; -#endif - -#include "build_geometry.h" - -typedef std::auto_ptr geom_ptr; - -static std::vector wkts; -static std::vector areas; - -static int excludepoly = 0; - -char *get_wkt_simple(osmNode *nodes, int count, int polygon) { - GeometryFactory gf; - std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - - try - { - for (int i = 0; i < count ; i++) { - Coordinate c; - c.x = nodes[i].lon; - c.y = nodes[i].lat; - coords->add(c, 0); - } - - geom_ptr geom; - if (polygon && (coords->getSize() >= 4) && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0)))) { - std::auto_ptr shell(gf.createLinearRing(coords.release())); - geom = geom_ptr(gf.createPolygon(shell.release(), new std::vector)); - if (!geom->isValid()) { - if (excludepoly) { - return NULL; - } else { - geom = geom_ptr(geom->buffer(0)); - } - } - geom->normalize(); // Fix direction of ring - } else { - if (coords->getSize() < 2) - return NULL; - geom = geom_ptr(gf.createLineString(coords.release())); - } - - WKTWriter wktw; - std::string wkt = wktw.write(geom.get()); - return strdup(wkt.c_str()); - } - catch (std::bad_alloc) - { - std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl; - std::cerr << "Try in slim mode, using -s parameter." << std::endl; - return NULL; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing way" << std::endl; - return NULL; - } -} - - -size_t get_wkt_split(osmNode *nodes, int count, int polygon, double split_at) { - GeometryFactory gf; - std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - double area; - WKTWriter wktw; - size_t wkt_size = 0; - - try - { - for (int i = 0; i < count ; i++) { - Coordinate c; - c.x = nodes[i].lon; - c.y = nodes[i].lat; - coords->add(c, 0); - } - - geom_ptr geom; - if (polygon && (coords->getSize() >= 4) && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0)))) { - std::auto_ptr shell(gf.createLinearRing(coords.release())); - geom = geom_ptr(gf.createPolygon(shell.release(), new std::vector)); - if (!geom->isValid()) { - if (excludepoly) { - return 0; - } else { - geom = geom_ptr(geom->buffer(0)); - } - } - geom->normalize(); // Fix direction of ring - area = geom->getArea(); - std::string wkt = wktw.write(geom.get()); - wkts.push_back(wkt); - areas.push_back(area); - wkt_size++; - } else { - if (coords->getSize() < 2) - return 0; - - double distance = 0; - std::auto_ptr segment; - segment = std::auto_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(coords->getAt(0)); - for(unsigned i=1; igetSize(); i++) { - segment->add(coords->getAt(i)); - distance += coords->getAt(i).distance(coords->getAt(i-1)); - if ((distance >= split_at) || (i == coords->getSize()-1)) { - geom = geom_ptr(gf.createLineString(segment.release())); - std::string wkt = wktw.write(geom.get()); - wkts.push_back(wkt); - areas.push_back(0); - wkt_size++; - distance=0; - segment = std::auto_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(coords->getAt(i)); - } - } - } - - } - catch (std::bad_alloc) - { - std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl; - std::cerr << "Try in slim mode, using -s parameter." << std::endl; - wkt_size = 0; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing way" << std::endl; - wkt_size = 0; - } - return wkt_size; -} - - -char * get_wkt(size_t index) -{ -// return wkts[index].c_str(); - char *result; - result = (char*) std::malloc( wkts[index].length() + 1); - // At least give some idea of why we about to seg fault - if (!result) std::cerr << std::endl << "Unable to allocate memory: " << (wkts[index].length() + 1) << std::endl; - std::strcpy(result, wkts[index].c_str()); - return result; -} - -double get_area(size_t index) -{ - return areas[index]; -} - -void clear_wkts() -{ - wkts.clear(); - areas.clear(); -} - -static int coords2nodes(CoordinateSequence * coords, struct osmNode ** nodes) { - size_t num_coords; - size_t i; - Coordinate coord; - - num_coords = coords->getSize(); - *nodes = (struct osmNode *) malloc(num_coords * sizeof(struct osmNode)); - - for (i = 0; i < num_coords; i++) { - coord = coords->getAt(i); - (*nodes)[i].lon = coord.x; - (*nodes)[i].lat = coord.y; - } - return num_coords; -} - -int parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int * polygon) { - GeometryFactory gf; - WKTReader reader(&gf); - std::string wkt_string(wkt); - Geometry * geometry; - const Geometry * subgeometry; - GeometryCollection * gc; - CoordinateSequence * coords; - size_t num_geometries; - size_t i; - - *polygon = 0; - try { - geometry = reader.read(wkt_string); - switch (geometry->getGeometryTypeId()) { - // Single geometries - case GEOS_POLYGON: - // Drop through - case GEOS_LINEARRING: - *polygon = 1; - // Drop through - case GEOS_POINT: - // Drop through - case GEOS_LINESTRING: - *xnodes = (struct osmNode **) malloc(2 * sizeof(struct osmNode *)); - *xcount = (int *) malloc(sizeof(int)); - coords = geometry->getCoordinates(); - (*xcount)[0] = coords2nodes(coords, &((*xnodes)[0])); - (*xnodes)[1] = NULL; - delete coords; - break; - // Geometry collections - case GEOS_MULTIPOLYGON: - *polygon = 1; - // Drop through - case GEOS_MULTIPOINT: - // Drop through - case GEOS_MULTILINESTRING: - gc = dynamic_cast(geometry);; - num_geometries = gc->getNumGeometries(); - *xnodes = (struct osmNode **) malloc((num_geometries + 1) * sizeof(struct osmNode *)); - *xcount = (int *) malloc(num_geometries * sizeof(int)); - for (i = 0; i < num_geometries; i++) { - subgeometry = gc->getGeometryN(i); - coords = subgeometry->getCoordinates(); - (*xcount)[i] = coords2nodes(coords, &((*xnodes)[i])); - delete coords; - } - (*xnodes)[i] = NULL; - break; - default: - std::cerr << std::endl << "unexpected object type while processing PostGIS data" << std::endl; - delete geometry; - return -1; - } - delete geometry; - } catch (...) { - std::cerr << std::endl << "Exception caught parsing PostGIS data" << std::endl; - return -1; - } - return 0; -} - -struct polygondata -{ - Polygon* polygon; - LinearRing* ring; - double area; - int iscontained; - unsigned containedbyid; -}; - -static int polygondata_comparearea(const void* vp1, const void* vp2) -{ - const polygondata* p1 = (const polygondata*)vp1; - const polygondata* p2 = (const polygondata*)vp2; - - if (p1->area == p2->area) return 0; - if (p1->area > p2->area) return -1; - return 1; -} - -size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int make_polygon, int enable_multi, double split_at) { - size_t wkt_size = 0; - std::auto_ptr > lines(new std::vector); - GeometryFactory gf; - geom_ptr geom; -#ifdef HAS_PREPARED_GEOMETRIES - geos::geom::prep::PreparedGeometryFactory pgf; -#endif - - try - { - for (int c=0; xnodes[c]; c++) { - std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - for (int i = 0; i < xcount[c]; i++) { - struct osmNode *nodes = xnodes[c]; - Coordinate c; - c.x = nodes[i].lon; - c.y = nodes[i].lat; - coords->add(c, 0); - } - if (coords->getSize() > 1) { - geom = geom_ptr(gf.createLineString(coords.release())); - lines->push_back(geom.release()); - } - } - - //geom_ptr segment(0); - geom_ptr mline (gf.createMultiLineString(lines.release())); - //geom_ptr noded (segment->Union(mline.get())); - LineMerger merger; - //merger.add(noded.get()); - merger.add(mline.get()); - std::auto_ptr > merged(merger.getMergedLineStrings()); - WKTWriter writer; - - // Procces ways into lines or simple polygon list - polygondata* polys = new polygondata[merged->size()]; - - unsigned totalpolys = 0; - for (unsigned i=0 ;i < merged->size(); ++i) - { - std::auto_ptr pline ((*merged ) [i]); - if (make_polygon && pline->getNumPoints() > 3 && pline->isClosed()) - { - polys[totalpolys].polygon = gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0); - polys[totalpolys].ring = gf.createLinearRing(pline->getCoordinates()); - polys[totalpolys].area = polys[totalpolys].polygon->getArea(); - polys[totalpolys].iscontained = 0; - polys[totalpolys].containedbyid = 0; - if (polys[totalpolys].area > 0.0) - totalpolys++; - else { - delete(polys[totalpolys].polygon); - delete(polys[totalpolys].ring); - } - } - else - { - //std::cerr << "polygon(" << osm_id << ") is no good: points(" << pline->getNumPoints() << "), closed(" << pline->isClosed() << "). " << writer.write(pline.get()) << std::endl; - double distance = 0; - std::auto_ptr segment; - segment = std::auto_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(pline->getCoordinateN(0)); - for(unsigned i=1; igetNumPoints(); i++) { - segment->add(pline->getCoordinateN(i)); - distance += pline->getCoordinateN(i).distance(pline->getCoordinateN(i-1)); - if ((distance >= split_at) || (i == pline->getNumPoints()-1)) { - geom = geom_ptr(gf.createLineString(segment.release())); - std::string wkt = writer.write(geom.get()); - wkts.push_back(wkt); - areas.push_back(0); - wkt_size++; - distance=0; - segment = std::auto_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); - segment->add(pline->getCoordinateN(i)); - } - } - //std::string text = writer.write(pline.get()); - //wkts.push_back(text); - //areas.push_back(0.0); - //wkt_size++; - } - } - - if (totalpolys) - { - qsort(polys, totalpolys, sizeof(polygondata), polygondata_comparearea); - - unsigned toplevelpolygons = 0; - int istoplevelafterall; - - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained != 0) continue; - toplevelpolygons++; -#ifdef HAS_PREPARED_GEOMETRIES - const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon); -#endif - - for (unsigned j=i+1; j < totalpolys; ++j) - { -#ifdef HAS_PREPARED_GEOMETRIES - // Does preparedtoplevelpolygon contain the smaller polygon[j]? - if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon)) -#else - // Does polygon[i] contain the smaller polygon[j]? - if (polys[j].containedbyid == 0 && polys[i].polygon->contains(polys[j].polygon)) -#endif - { - // are we in a [i] contains [k] contains [j] situation - // which would actually make j top level - istoplevelafterall = 0; - for (unsigned k=i+1; k < j; ++k) - { - if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon)) - { - istoplevelafterall = 1; - break; - } -#if 0 - else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon)) - { - // FIXME: This code does not work as intended - // It should be setting the polys[k].ring in order to update this object - // but the value of polys[k].polygon calculated is normally NULL - - // Add polygon this polygon (j) to k since they intersect - // Mark ourselfs to be dropped (2), delete the original k - Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon); - delete(polys[k].polygon); - polys[k].polygon = dynamic_cast(polyunion); - polys[j].iscontained = 2; // Drop - istoplevelafterall = 2; - break; - - } -#endif - } - if (istoplevelafterall == 0) - { - polys[j].iscontained = 1; - polys[j].containedbyid = i; - } - } - } -#ifdef HAS_PREPARED_GEOMETRIES - pgf.destroy(preparedtoplevelpolygon); -#endif - } - // polys now is a list of ploygons tagged with which ones are inside each other - - // List of polygons for multipolygon - std::auto_ptr > polygons(new std::vector); - - // For each top level polygon create a new polygon including any holes - for (unsigned i=0 ;i < totalpolys; ++i) - { - if (polys[i].iscontained != 0) continue; - - // List of holes for this top level polygon - std::auto_ptr > interior(new std::vector); - for (unsigned j=i+1; j < totalpolys; ++j) - { - if (polys[j].iscontained == 1 && polys[j].containedbyid == i) - { - interior->push_back(polys[j].ring); - } - } - - Polygon* poly(gf.createPolygon(polys[i].ring, interior.release())); - poly->normalize(); - polygons->push_back(poly); - } - - // Make a multipolygon if required - if ((toplevelpolygons > 1) && enable_multi) - { - geom_ptr multipoly(gf.createMultiPolygon(polygons.release())); - if (!multipoly->isValid() && (excludepoly == 0)) { - multipoly = geom_ptr(multipoly->buffer(0)); - } - multipoly->normalize(); - - if ((excludepoly == 0) || (multipoly->isValid())) - { - std::string text = writer.write(multipoly.get()); - wkts.push_back(text); - areas.push_back(multipoly->getArea()); - wkt_size++; - } - } - else - { - for(unsigned i=0; i(polygons->at(i)); - if (!poly->isValid() && (excludepoly == 0)) { - poly = dynamic_cast(poly->buffer(0)); - poly->normalize(); - } - if ((excludepoly == 0) || (poly->isValid())) - { - std::string text = writer.write(poly); - wkts.push_back(text); - areas.push_back(poly->getArea()); - wkt_size++; - } - delete(poly); - } - } - } - - for (unsigned i=0; i < totalpolys; ++i) - { - delete(polys[i].polygon); - } - delete[](polys); - } - catch (std::exception& e) - { - std::cerr << std::endl << "Standard exception processing way_id "<< osm_id << ": " << e.what() << std::endl; - wkt_size = 0; - } - catch (...) - { - std::cerr << std::endl << "Exception caught processing way id=" << osm_id << std::endl; - wkt_size = 0; - } - - return wkt_size; -} - -void exclude_broken_polygon () -{ - excludepoly = 1; -} diff -Nru osm2pgsql-0.82.0/build_geometry.h osm2pgsql-0.88.1/build_geometry.h --- osm2pgsql-0.82.0/build_geometry.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/build_geometry.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,47 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef BUILD_GEOMETRY_H -#define BUILD_GEOMETRY_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "osmtypes.h" - -int parse_wkt(const char * wkt, struct osmNode *** xnodes, int ** xcount, int * polygon); - -char *get_wkt_simple(struct osmNode *, int count, int polygon); -size_t get_wkt_split(struct osmNode *, int count, int polygon, double split_at); - -char* get_wkt(size_t index); -double get_area(size_t index); -size_t build_geometry(osmid_t osm_id, struct osmNode **xnodes, int *xcount, int make_polygon, int enable_multi, double split_at); -void clear_wkts(); -void exclude_broken_polygon (); - -#ifdef __cplusplus -} -#endif - -#endif diff -Nru osm2pgsql-0.82.0/configure.ac osm2pgsql-0.88.1/configure.ac --- osm2pgsql-0.82.0/configure.ac 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/configure.ac 2015-08-15 05:28:11.000000000 +0000 @@ -1,11 +1,14 @@ dnl Process this file with autoconf to produce a configure script. -AC_INIT(osm2pgsql, 0.82.0) +AC_INIT(osm2pgsql, 0.88.1) dnl Required autoconf version AC_PREREQ(2.61) AX_CONFIG_NICE +dnl Allow maintainer mode to be disabled (debian build scripts do this) +AM_MAINTAINER_MODE([enable]) + dnl use automake to generate standard Makefiles AM_INIT_AUTOMAKE([1.9.6 dist-bzip2 std-options check-news]) @@ -34,8 +37,8 @@ AC_MSG_ERROR([Could not find a c++ compiler]); fi -dnl AX_CFLAGS_WARN_ALL -dnl AX_CXXFLAGS_WARN_ALL +AX_CFLAGS_WARN_ALL +AX_CXXFLAGS_WARN_ALL dnl Make sure we have libtool installed AM_PROG_LIBTOOL @@ -46,17 +49,6 @@ AC_CHECK_FUNC(lseek64,[AC_DEFINE(HAVE_LSEEK64, [1], [lseek64 is present])],[AX_COMPILE_CHECK_SIZEOF(off_t)]) AC_CHECK_FUNCS([posix_fallocate posix_fadvise sync_file_range fork]) - -dnl legacy 32bit ID mode -AC_ARG_ENABLE([64bit-ids], - AS_HELP_STRING([--disable-64bit-ids], [Disable 64bit IDs for OSM IDs]), - [ if test "$enableval" = "yes" - then - AC_DEFINE(OSMID64, [1], [Enable 64bit OSM IDs]) - fi - ], [ AC_DEFINE(OSMID64, [1], [Enable 64bit OSM IDs])]) - - dnl Check for libxml2 library AX_LIB_XML2 if test "$HAVE_XML2" = "no" @@ -79,11 +71,7 @@ fi dnl Check for Geos library -AX_LIB_GEOS -if test "x$GEOS_VERSION" = "x" -then - AC_MSG_ERROR([geos library not found]); -fi +AX_LIB_GEOS([3.1]) dnl Check for Proj library AX_LIB_PROJ @@ -118,9 +106,48 @@ dnl Check for pthread library AX_PTHREAD(,[AC_MSG_ERROR([no])]) +dnl Check for Boost libraries +AX_BOOST_BASE([1.48], , [AC_MSG_ERROR([cannot find Boost libraries, which are are required for building osm2pgsql. Please install libboost-dev.])]) + +AX_BOOST_SYSTEM +AX_BOOST_FILESYSTEM +AX_BOOST_THREAD +if test "x$BOOST_SYSTEM_LIB" = "x" -o "x$BOOST_FILESYSTEM_LIB" = "x" -o "x$BOOST_THREAD_LIB" = "x" +then + AC_MSG_ERROR([One or more of the mandatory Boost libraries not found.]) +fi + +dnl Boost json parser in 1.49 has a bug when compiled with C++11 +dnl see https://svn.boost.org/trac/boost/ticket/6785 +AC_ARG_WITH([cxx11], + [AS_HELP_STRING([--without-cxx11], + [do not check for C++11-capable compiler (for testing only)])], + [], + [with_cxx11=yes]) + +if test "x$with_cxx11" = "xyes"; then + AX_BOOST_BASE([1.50], [ AX_CXX_COMPILE_STDCXX_11(,optional) ], []) +fi + +dnl Check for Lua libraries and headers +AX_PROG_LUA([5.0],[],[ + AX_LUA_HEADERS([ + AX_LUA_LIBS([ + AC_DEFINE([HAVE_LUA], [1], [Requirements for lua are met]) + HAVE_LUA=yes + ],[AC_MSG_WARN([cannot find Lua libs])]) + ],[AC_MSG_WARN([cannot find Lua includes])]) +],[AC_MSG_WARN([cannot find Lua interpreter])]) + +dnl Enable fixed point +AC_ARG_WITH([fixed-point], + [AS_HELP_STRING([--without-fixed-point], + [use double instead of fixed point floats for coordinates])], + [], + [AC_DEFINE([FIXED_POINT], [1], [Store +-20,000km Mercator co-ordinates as fixed point 32bit number with maximum precision])]) dnl Generate Makefile -AC_OUTPUT(Makefile legacy/Makefile) +AC_OUTPUT(Makefile) if test "$BUILD_READER_PBF" != "yes" then @@ -131,3 +158,14 @@ Look for packages named: libprotobuf-c0-dev protobuf-c-compiler ]) fi + +if test "$HAVE_LUA" != "yes" +then + AC_MSG_WARN([ +lua libraries not found. You will NOT be able to use lua scripts for tag transform. + +To enable lua support, the lua interpreter and libraries are required. +Look for packages named: lua5.2 liblua5.2-dev +]) +fi + diff -Nru osm2pgsql-0.82.0/CONTRIBUTING.md osm2pgsql-0.88.1/CONTRIBUTING.md --- osm2pgsql-0.82.0/CONTRIBUTING.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/CONTRIBUTING.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,103 @@ +# Osm2pgsql contribution guidelines + +## Workflow + +We operate the "Fork & Pull" model explained at + +https://help.github.com/articles/using-pull-requests + +You should fork the project into your own repo, create a topic branch +there and then make one or more pull requests back to the openstreetmap repository. +Your pull requests will then be reviewed and discussed. + +## History + +To understand the osm2pgsql code, it helps to know some history on it. Osm2pgsql +was written in C in 2007 as a port of an older Python utility. In 2014 it was +ported to C++ by MapQuest and the last C version was released as 0.86.0. In it's +time, it has had varying contribution activity, including times with no +maintainer or active developers. + +Parts of the codebase still clearly show their C origin and could use rewriting +in modern C++, making use of data structures in the standard library. + +## Versioning + +Osm2pgsql uses a X.Y.Z version number, where Y tells you if you are on a stable +or development series. Like the Linux Kernel, even numbers are stable and +development versions are odd. + +Bugs and known issues are fixed on the main branch only. Exceptions may be made +for easy bug fixes, or if a patch backporting a fix is provided. + +## Code style + +The current codebase is a mix of styles, but new code should be written in the +[K&R 1TBS style](https://en.wikipedia.org/wiki/Indent_style#Variant:_1TBS) with +4 spaces indentation. Tabs should never be used in the C++ code. + +e.g. + +``` +int main(int argc, char *argv[]) +{ + ... + while (x == y) { + something(); + somethingelse(); + + if (some_error) { + do_correct(); + } else { + continue_as_usual(); + } + } + + finalthing(); + ... +} +``` + +Names should use underscores, not camel case, with class/struct names ending in `_t`. + +## Platforms targeted + +Ideally osm2pgsql should compile on Linux, OS X, FreeBSD and Windows. It is +actively tested on Debian, Ubuntu and FreeBSD by the maintainers. + +## Testing + +The code also comes with a suite of tests which can be run by +executing ``make check``. + +Most of these tests depend on being able to set up a database and run osm2pgsql +against it. You need to ensure that PostgreSQL is running and that your user is +a superuser of that system. To do that, run: + +```sh +sudo -u postgres createuser -s $USER +sudo mkdir -p /tmp/psql-tablespace +sudo chown postgres.postgres /tmp/psql-tablespace +psql -c "CREATE TABLESPACE tablespacetest LOCATION '/tmp/psql-tablespace'" postgres +``` + +Once this is all set up, all the tests should run (no SKIPs), and pass +(no FAILs). If you encounter a failure, you can find more information +by looking in the `test-suite.log`. If you find something which seems +to be a bug, please check to see if it is a known issue at +https://github.com/openstreetmap/osm2pgsql/issues and, if it's not +already known, report it there. + +If running the tests in a virtual machine, allocate sufficient disk space for a +20GB flat nodes file. + +### Performance Testing + +If performance testing with a full planet import is required, indicate what +needs testing in a pull request. + +## Maintainers + +The current maintainers of osm2pgsql are [Sarah Hoffmann](https://github.com/lonvia/) +and [Paul Norman](https://github.com/pnorman/). Sarah has more experience with +the gazetteer backend and Paul with the pgsql and multi backends. diff -Nru osm2pgsql-0.82.0/cygpkg.sh osm2pgsql-0.88.1/cygpkg.sh --- osm2pgsql-0.82.0/cygpkg.sh 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/cygpkg.sh 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Author: Jason Huntley +# Email: onepremise@gmail.com +# Description: Cygwin Package script +# +# Change Log +# +# Date Description Initials +#------------------------------------------------------------- +# 04-11-13 Initial Coding JAH +#============================================================= + +if [ ! -e "cygwin-package" ]; then + mkdir cygwin-package +fi + +echo +echo Copying Executable... +echo + +cp -rfv default.style cygwin-package || { stat=$?; echo "Packaging failed, aborting" >&2; exit $stat; } +cp -rfv 900913.sql cygwin-package || { stat=$?; echo "Packaging failed, aborting" >&2; exit $stat; } +cp -rfv README cygwin-package || { stat=$?; echo "Packaging failed, aborting" >&2; exit $stat; } +cp -rfv .libs/osm2pgsql.exe cygwin-package || { stat=$?; echo "Packaging failed, aborting" >&2; exit $stat; } + +echo +echo Copying Dependent Libraries... +echo + +cp -rfv /bin/cygcrypt*.dll cygwin-package +cp -rfv /bin/cyggcc*.dll cygwin-package +cp -rfv /usr/local/bin/cyggeos*.dll cygwin-package +cp -rfv /bin/cygiconv*.dll cygwin-package +cp -rfv /bin/cygintl*.dll cygwin-package +cp -rfv /bin/cyglber*.dll cygwin-package +cp -rfv /bin/cygldap*.dll cygwin-package +cp -rfv /bin/cyglzma*.dll cygwin-package +cp -rfv /bin/cygpq*.dll cygwin-package +cp -rfv /usr/local/bin/cygproj*.dll cygwin-package +cp -rfv /usr/local/bin/cygproto*.dll cygwin-package +cp -rfv /bin/cygsasl*.dll cygwin-package +cp -rfv /bin/cygssl*.dll cygwin-package +cp -rfv /bin/cygstdc++**.dll cygwin-package +cp -rfv /bin/cygwin*.dll cygwin-package +cp -rfv /bin/cygxml2*.dll cygwin-package +cp -rfv /bin/cygz*.dll cygwin-package + +echo +echo Creating Archive... +echo + +zip -r9 cygwin-package.zip cygwin-package + +echo +echo Packaging Complete. +echo + +exit 0 diff -Nru osm2pgsql-0.82.0/debian/changelog osm2pgsql-0.88.1/debian/changelog --- osm2pgsql-0.82.0/debian/changelog 2013-12-18 07:47:00.000000000 +0000 +++ osm2pgsql-0.88.1/debian/changelog 2015-10-12 21:55:20.000000000 +0000 @@ -1,13 +1,172 @@ -osm2pgsql (0.82.0-1git1) trusty; urgency=low +osm2pgsql (0.88.1-1ubuntu1) trusty; urgency=medium - Upload current Debian git head for the postgis transition. + * Move to trusty. - [ Bas Couwenberg ] - * Team upload. + -- Hiroshi Miura Tue, 13 Oct 2015 06:54:22 +0900 + +osm2pgsql (0.88.1-1) unstable; urgency=medium + + * New upstream release. + * Update watch file to handle other tar extensions. + * Update Vcs-Browser URL to use HTTPS. + + -- Bas Couwenberg Fri, 28 Aug 2015 11:51:55 +0200 + +osm2pgsql (0.88.0-1) unstable; urgency=medium + + * Fix filenamemangle in watch file for non pre-releases. + * New upstream release. + * Add lintian override for spelling-error-in-binary false positive. + * Move from experimental to unstable. + + -- Bas Couwenberg Wed, 15 Jul 2015 08:05:18 +0200 + +osm2pgsql (0.88.0~rc1-1~exp1) experimental; urgency=medium + + * Support pre-releases in watch file. + * New upstream release candidate. + * Add additional contributors to copyright file. + * Also include CONTRIBUTING.md in docs. + + -- Bas Couwenberg Sat, 11 Jul 2015 10:31:41 +0200 + +osm2pgsql (0.87.4-1) unstable; urgency=medium + + * New upstream release. + * Update copyright file, changes: + - Drop license & copyright for rb.{c,h}pp, removed upstream. + + -- Bas Couwenberg Mon, 06 Jul 2015 08:37:56 +0200 + +osm2pgsql (0.87.3-3) unstable; urgency=medium + + * Use --without-lockfree configure option on problematic architectures, + still required despite upstream claims. + + -- Bas Couwenberg Fri, 01 May 2015 11:22:39 +0200 + +osm2pgsql (0.87.3-2) unstable; urgency=medium + + * Drop --without-lockfree configure option, no longer needed. + + -- Bas Couwenberg Thu, 30 Apr 2015 23:56:01 +0200 + +osm2pgsql (0.87.3-1) unstable; urgency=medium + + * New upstream release. + * Change Upstream-Contact to OpenStreetMap Developers. + + -- Bas Couwenberg Thu, 30 Apr 2015 09:09:57 +0200 + +osm2pgsql (0.87.2-1) unstable; urgency=medium + + * Move from experimental to unstable. + + -- Bas Couwenberg Sun, 26 Apr 2015 20:12:07 +0200 + +osm2pgsql (0.87.2-1~exp3) experimental; urgency=medium + + * Also use --without-lockfree configure option on sparc & alpha. + + -- Bas Couwenberg Sat, 07 Mar 2015 17:05:36 +0100 + +osm2pgsql (0.87.2-1~exp2) experimental; urgency=medium + + * Use --without-lockfree configure option only on architectures where the + build fails otherwise. The option introduces a performance penalty + undesirable on the primary architectures. + + -- Bas Couwenberg Tue, 24 Feb 2015 19:41:52 +0100 + +osm2pgsql (0.87.2-1~exp1) experimental; urgency=medium + + * New upstream release. + * Use --without-lockfree configure option instead of patching the + source. + * Include additional documentation. + * Update my email to @debian.org address. + + -- Bas Couwenberg Tue, 24 Feb 2015 09:11:40 +0100 + +osm2pgsql (0.87.1-1~exp1) experimental; urgency=medium + + * New upstream release. + * Update Vcs-Browser URL to use cgit instead of gitweb. + + -- Bas Couwenberg Tue, 30 Dec 2014 08:50:38 +0100 + +osm2pgsql (0.87.0-1~exp2) experimental; urgency=medium + + * Add patch to not use Boost lockfree queues even for Boost >= 1.53, + hopefully fixes the FTBFS on arm64, armel, powerpc, ppc64el & s390x. + + -- Bas Couwenberg Fri, 12 Dec 2014 01:38:00 +0100 + +osm2pgsql (0.87.0-1~exp1) experimental; urgency=medium + + * New upstream release. + * Add upstream metadata. + * Update watch file to rename downloaded tarball. + * Add Boost build dependencies for C++ build. + * Enable verbose test output. + * Ignore test failures, some tests require a configured postgres + database. + * Update copyright file, changes: + - Add Upstream-Name header + - Update Files sections for C++ files + - Update Source URL to match watch file URL + - Fix geos-fallback license, GEOS uses LGPL-2.1+ not LGPL-3+ + - Add Copyright & License for win_fsync.h + + -- Bas Couwenberg Wed, 10 Dec 2014 20:13:03 +0100 + +osm2pgsql (0.86.0-1) unstable; urgency=medium + + * New upstream release. + * Remove 00-fix_build.patch, applied upstream. + * Remove 01-hyphen-used-as-minus-sign.patch, applied upstream. + * Remove 02-spelling-error-in-manpage.patch, applied upstream. + * Remove 03-fix-build-for-protobuf-c-1.0.0.patch, applied upstream. + * Remove 04-api-changes-for-protobuf-c-1.0.0.patch, applied upstream. + * Add python build dependencies for updated regression test script. + * Update docs to install README.md. + * Bump Standards-Version to 3.9.6, no changes. + + -- Bas Couwenberg Sat, 25 Oct 2014 13:15:52 +0200 + +osm2pgsql (0.84.0-3) unstable; urgency=medium + + * Only recommend postgis, now that postgis recommends the postgresql + specific shared library. + (closes: #757972) + + -- Bas Couwenberg Sun, 17 Aug 2014 15:11:05 +0200 + +osm2pgsql (0.84.0-2) unstable; urgency=medium + + * Add patches for protobuf-c 1.0.0 support. + Thanks to Robert Edmonds for the patches. + (closes: #755553) + + -- Bas Couwenberg Sat, 26 Jul 2014 13:33:01 +0200 + +osm2pgsql (0.84.0-1) unstable; urgency=low + + * New upstream release. + * Add myself to Uploaders. * Update postgis Recommends to postgresql-9.3-postgis-2.1. - (closes: #732413) + (closes: #732415) + * Add gbp.conf to use pristine-tar by default. + * Refresh patches. + * Drop get-orig-source target, upstream moved from OSM SVN to GitHub. + * Add build dependency on liblua5.2-dev and lua5.2. + * Append CPPFLAGS to CFLAGS & CXXFLAGS to use -D_FORTIFY_SOURCE=2. + * Add patch to fix hyphen-used-as-minus-sign warnings. + * Add patch to fix spelling-error-in-manpage warning. + * Bump Standards-Version to 3.9.5, no changes. + * Enable parallel builds. - -- Martin Pitt Wed, 18 Dec 2013 08:46:43 +0100 + -- Bas Couwenberg Wed, 30 Apr 2014 13:05:29 +0200 osm2pgsql (0.82.0-1) unstable; urgency=low diff -Nru osm2pgsql-0.82.0/debian/control osm2pgsql-0.88.1/debian/control --- osm2pgsql-0.82.0/debian/control 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/control 2015-08-28 09:51:34.000000000 +0000 @@ -2,21 +2,31 @@ Maintainer: Debian GIS Project Uploaders: Francesco Paolo Lovergine , David Paleino , - Andreas Tille + Andreas Tille , + Bas Couwenberg Section: utils Priority: optional Build-Depends: debhelper (>= 9), dh-autoreconf, + libboost-dev, + libboost-system-dev, + libboost-filesystem-dev, + libboost-thread-dev, libbz2-dev, zlib1g-dev, + libgeos-dev, libgeos++-dev, libpq-dev, libxml2-dev, libproj-dev, - libprotobuf-c0-dev (>= 0.14), - protobuf-c-compiler -Standards-Version: 3.9.4 -Vcs-Browser: http://anonscm.debian.org/gitweb/?p=pkg-grass/osm2pgsql.git + libprotobuf-c-dev | libprotobuf-c0-dev (>= 0.14), + protobuf-c-compiler, + liblua5.2-dev, + lua5.2, + python, + python-psycopg2 +Standards-Version: 3.9.6 +Vcs-Browser: https://anonscm.debian.org/cgit/pkg-grass/osm2pgsql.git Vcs-Git: git://anonscm.debian.org/pkg-grass/osm2pgsql.git Homepage: http://wiki.openstreetmap.org/wiki/Osm2pgsql @@ -24,10 +34,7 @@ Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} -Recommends: postgis, - postgresql-9.3-postgis-2.1 -Suggests: josm, - gosmore +Recommends: postgis Description: OpenStreetMap data to PostgreSQL converter Convert OSM planet snapshot data to SQL suitable for loading into a PostgreSQL database with PostGIS geospatial extensions. This diff -Nru osm2pgsql-0.82.0/debian/copyright osm2pgsql-0.88.1/debian/copyright --- osm2pgsql-0.82.0/debian/copyright 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/copyright 2015-07-20 07:08:57.000000000 +0000 @@ -1,30 +1,39 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0 -Upstream-Contact: Kai Krueger -Source: https://github.com/openstreetmap/osm2pgsql/tags +Upstream-Name: osm2pgsql +Upstream-Contact: OpenStreetMap Developers +Source: https://github.com/openstreetmap/osm2pgsql/releases Files: * Copyright: © 2007-2008, Artem Pavlenko - © 2006-2008, Jon Burgess -License: GPL-2+ - -Files: debian/* -Copyright: © 2007-2009, Andreas Putzo - © 2010-2012, David Paleino + © 2006-2008, Jon Burgess +Comment: osm2pgsql was written by Jon Burgess, Artem Pavlenko, + Martijn van Oosterhout, Sarah Hoffmann, Kai Krueger, Frederik Ramm, + Brian Quinion, Matt Amos, Kevin Kreiser, Paul Norman and other + OpenStreetMap project members. License: GPL-2+ Files: geos-fallback/* Copyright: © 2006, Refractions Research Inc. -License: LGPL-3+ - -Files: rb.c -Copyright: © 1998-2002, 2004, Free Software Foundation, Inc. -License: GPL-2+ +License: LGPL-2.1+ -Files: sprompt.c +Files: sprompt.cpp Copyright: © 1994, The Regents of the University of California - © 1996-2006, PostgreSQL Global Development Group + © 1996-2006, PostgreSQL Global Development Group License: other +Files: wildcmp.cpp +Copyright: © 2002, Jim Kent +License: GPL-2+ + +Files: win_fsync.h +Copyright: 2008-2014, Free Software Foundation, Inc. +License: LGPL-2.1+ + +Files: debian/* +Copyright: © 2007-2009, Andreas Putzo + © 2010-2012, David Paleino +License: GPL-2+ + License: GPL-2+ This package is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -60,19 +69,18 @@ ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. -License: LGPL-3+ - This package is free software; you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 3 of the License, or - (at your option) any later version. +License: LGPL-2.1+ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. . - This package is distributed in the hope that it will be useful, + This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. . - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . - . - On Debian systems, the complete text of the GNU Lesser General Public License - version 3 or later can be found in `/usr/share/common-licenses/LGPL-3'. + On Debian systems, the full text of the GNU Lesser General Public + License version 2.1 can be found in the file + `/usr/share/common-licenses/LGPL-2.1'. + diff -Nru osm2pgsql-0.82.0/debian/docs osm2pgsql-0.88.1/debian/docs --- osm2pgsql-0.82.0/debian/docs 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/docs 2015-07-20 07:08:57.000000000 +0000 @@ -1,4 +1,6 @@ AUTHORS +CONTRIBUTING.md NEWS -README +README.md TODO +docs/*.md diff -Nru osm2pgsql-0.82.0/debian/gbp.conf osm2pgsql-0.88.1/debian/gbp.conf --- osm2pgsql-0.82.0/debian/gbp.conf 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/debian/gbp.conf 2015-08-17 16:28:39.000000000 +0000 @@ -0,0 +1,16 @@ +[DEFAULT] + +# The default name for the upstream branch is "upstream". +# Change it if the name is different (for instance, "master"). +upstream-branch = upstream + +# The default name for the Debian branch is "master". +# Change it if the name is different (for instance, "debian/unstable"). +debian-branch = master + +# git-import-orig uses the following names for the upstream tags. +# Change the value if you are not using git-import-orig +upstream-tag = upstream/%(version)s + +# Always use pristine-tar. +pristine-tar = True diff -Nru osm2pgsql-0.82.0/debian/install osm2pgsql-0.88.1/debian/install --- osm2pgsql-0.82.0/debian/install 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/install 2015-02-24 08:05:11.000000000 +0000 @@ -1,2 +1,2 @@ -osm2pgsql /usr/bin/ +osm2pgsql /usr/bin default.style /usr/share/osm2pgsql diff -Nru osm2pgsql-0.82.0/debian/lintian-overrides osm2pgsql-0.88.1/debian/lintian-overrides --- osm2pgsql-0.82.0/debian/lintian-overrides 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/debian/lintian-overrides 2015-07-20 07:08:57.000000000 +0000 @@ -0,0 +1,3 @@ +# False positive +osm2pgsql: spelling-error-in-binary usr/bin/* ressize resize + diff -Nru osm2pgsql-0.82.0/debian/patches/00-fix_build.patch osm2pgsql-0.88.1/debian/patches/00-fix_build.patch --- osm2pgsql-0.82.0/debian/patches/00-fix_build.patch 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/patches/00-fix_build.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,16 +0,0 @@ -From: David Paleino -Subject: add missing ZLIB linkage -Last-Update: 2013-09-17 23:21:42 +0200 -Forwarded: no - ---- a/Makefile.am -+++ b/Makefile.am -@@ -38,7 +38,7 @@ endif - - osm2pgsqldir = $(datadir)/osm2pgsql - --AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @PROTOBUF_C_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' -+AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @PROTOBUF_C_CFLAGS@ @ZLIB_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' - AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -Igeos-fallback - - AM_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML2_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_C_LDFLAGS@ @PROTOBUF_C_LIBS@ diff -Nru osm2pgsql-0.82.0/debian/patches/series osm2pgsql-0.88.1/debian/patches/series --- osm2pgsql-0.82.0/debian/patches/series 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -00-fix_build.patch diff -Nru osm2pgsql-0.82.0/debian/rules osm2pgsql-0.88.1/debian/rules --- osm2pgsql-0.82.0/debian/rules 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/rules 2015-07-11 08:34:43.000000000 +0000 @@ -4,31 +4,26 @@ # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 -SVNREPO := http://svn.openstreetmap.org/applications/utils/export/osm2pgsql -VERSION := $(shell dpkg-parsechangelog | grep Version | cut -d' ' -f2) -UPVER := $(shell echo $(VERSION) | awk -F'+r' '{print $$1}') -SVNREV := $(shell echo $(VERSION) | awk -F'+r' '{print $$2}' | cut -d- -f1) -SUFFIX := +r$(SVNREV) +DEB_HOST_ARCH ?= $(shell dpkg-architecture -qDEB_HOST_ARCH) + +# Verbose test output +export VERBOSE=1 + +CFLAGS += $(CPPFLAGS) +CXXFLAGS += $(CPPFLAGS) %: dh $@ \ - --with autoreconf + --with autoreconf \ + --parallel + +override_dh_auto_configure: + dh_auto_configure -- CFLAGS="$(CFLAGS)" CPPFLAGS="$(CPPFLAGS)" CXXFLAGS="$(CXXFLAGS)" + +override_dh_auto_test: + dh_auto_test || echo "Ignoring test failures" override_dh_compress: # clean up some naive file permissions dh_compress -X.php -X.sql -X.js -X.c -X.h -get-orig-source: - @echo Downloading osm2pgsql from ${SVNREPO} - svn export -r ${SVNREV} ${SVNREPO} osm2pgsql-${UPVER}${SUFFIX}/ - @echo Removing debian/ directory - rm -rf osm2pgsql-${UPVER}${SUFFIX}/debian/ - @echo Building snapshot tarball. - tar czvf osm2pgsql_${UPVER}${SUFFIX}.orig.tar.gz osm2pgsql-${UPVER}${SUFFIX} - @echo Cleaning up - rm -rf osm2pgsql-${UPVER}${SUFFIX} - @echo . - @echo To update debian/changelog type - @echo dch -v ${UPVER}${SUFFIX}-1 - @echo . - diff -Nru osm2pgsql-0.82.0/debian/upstream/metadata osm2pgsql-0.88.1/debian/upstream/metadata --- osm2pgsql-0.82.0/debian/upstream/metadata 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/debian/upstream/metadata 2015-04-30 07:08:54.000000000 +0000 @@ -0,0 +1,9 @@ +--- +Bug-Database: https://github.com/openstreetmap/osm2pgsql/issues +Bug-Submit: https://github.com/openstreetmap/osm2pgsql/issues/new +Contact: OpenStreetMap Developers +Donation: http://wiki.openstreetmap.org/wiki/Donations +Name: osm2pgsql +Registration: https://www.openstreetmap.org/user/new +Repository: https://github.com/openstreetmap/osm2pgsql.git +Repository-Browse: https://github.com/openstreetmap/osm2pgsql diff -Nru osm2pgsql-0.82.0/debian/watch osm2pgsql-0.88.1/debian/watch --- osm2pgsql-0.82.0/debian/watch 2013-12-18 07:46:14.000000000 +0000 +++ osm2pgsql-0.88.1/debian/watch 2015-08-22 16:40:59.000000000 +0000 @@ -1,2 +1,7 @@ version=3 -https://github.com/openstreetmap/osm2pgsql/tags .*/v*(\d[\d\.]+)\.tar\.gz +opts=\ +dversionmangle=s/\+(debian|dfsg|ds|deb)\d*$//,\ +uversionmangle=s/_/./g;s/(\d)[_\.\-\+]?((RC|rc|pre|dev|beta|alpha|b|a)[\-\.]?\d*)$/$1~$2/;s/RC/rc/,\ +filenamemangle=s/(?:.*?)?(?:rel|v|osm2psql)?[\-\_]?(\d\S+)\.(tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))/osm2psql-$1.$2/ \ +https://github.com/openstreetmap/osm2pgsql/releases \ +(?:.*?/)?(?:rel|v|osm2pqsql)?[\-\_]?(\d\S+)\.(?:tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) diff -Nru osm2pgsql-0.82.0/default.style osm2pgsql-0.88.1/default.style --- osm2pgsql-0.82.0/default.style 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/default.style 2015-08-15 05:28:11.000000000 +0000 @@ -1,124 +1,230 @@ -# This is the style file that matches the old version of osm2pgsql, which -# did not make distinctions between tags for nodes and for ways. There are a -# number of optimisations that can be applied here. Firstly, certain tags -# only apply to only nodes or only ways. By fixing this we reduce the amount -# of useless data loaded into the DB, which is a good thing. Possible -# optimisations for the future: - -# 1. Generate this file directly from the mapnik XML config, so it's always -# optimal - -# 2. Extend it so it can understand that highway=tertiary is for ways and -# highway=bus_stop is for nodes - -# Flags field isn't used much yet, expect if it contains the text "polygon" -# it indicates the shape is candidate for the polygon table. In the future I -# would like to be able to add directives like "nocache" which tells -# osm2pgsql that it is unlikely this node will be used by a way and so it -# doesn't need to be stored (eg coastline nodes). While in essence an -# optimisation hack, for --slim mode it doesn't matter if you're wrong, but -# in non-slim you might break something! - -# Also possibly an ignore flag, for things like "note" and "source" which -# can simply be deleted. (In slim mode this is, does not apply to non-slim -# obviously) - -# OsmType Tag DataType Flags -node,way note text delete # These tags can be long but are useless for rendering -node,way source text delete # This indicates that we shouldn't store them -node,way created_by text delete - -node,way access text linear -node,way addr:housename text linear -node,way addr:housenumber text linear -node,way addr:interpolation text linear -node,way admin_level text linear -node,way aerialway text linear -node,way aeroway text polygon -node,way amenity text nocache,polygon -node,way area text # hard coded support for area=1/yes => polygon is in osm2pgsql -node,way barrier text linear -node,way bicycle text nocache -node,way brand text linear -node,way bridge text linear -node,way boundary text linear -node,way building text polygon -node capital text linear -node,way construction text linear -node,way covered text linear -node,way culvert text linear -node,way cutting text linear -node,way denomination text linear -node,way disused text linear -node ele text linear -node,way embankment text linear -node,way foot text linear -node,way generator:source text linear -node,way harbour text polygon -node,way highway text linear -node,way historic text polygon -node,way horse text linear -node,way intermittent text linear -node,way junction text linear -node,way landuse text polygon -node,way layer text linear -node,way leisure text polygon -node,way lock text linear -node,way man_made text polygon -node,way military text polygon -node,way motorcar text linear -node,way name text linear -node,way natural text polygon # natural=coastline tags are discarded by a hard coded rule in osm2pgsql -node,way office text polygon -node,way oneway text linear -node,way operator text linear -node,way place text polygon -node poi text -node,way population text linear -node,way power text polygon -node,way power_source text linear -node,way public_transport text polygon -node,way railway text linear -node,way ref text linear -node,way religion text nocache -node,way route text linear -node,way service text linear -node,way shop text polygon -node,way sport text polygon -node,way surface text linear -node,way toll text linear -node,way tourism text polygon -node,way tower:type text linear -way tracktype text linear -node,way tunnel text linear -node,way water text polygon -node,way waterway text polygon -node,way wetland text polygon -node,way width text linear -node,way wood text linear -node,way z_order int4 linear # This is calculated during import -way way_area real # This is calculated during import - -# If you're interested in bicycle routes, you may want the following fields -# To make these work you need slim mode or the necessary data won't be remembered. -#way lcn_ref text linear -#way rcn_ref text linear -#way ncn_ref text linear -#way lcn text linear -#way rcn text linear -#way ncn text linear -#way lwn_ref text linear -#way rwn_ref text linear -#way nwn_ref text linear -#way lwn text linear -#way rwn text linear -#way nwn text linear -#way route_pref_color text linear -#way route_name text linear - -# The following entries can be used with the --extra-attributes option -# to include the username, userid, version & timstamp in the DB -#node,way osm_user text -#node,way osm_uid text -#node,way osm_version text -#node,way osm_timestamp text +# This is the default osm2pgsql .style file that comes with osm2pgsql. +# +# A .style file has 4 columns that define how OSM objects end up in tables in +# the database and what columns are created. It interacts with the command-line +# hstore options. +# +# Columns +# ======= +# +# OsmType: This is either "node", "way" or "node,way" and indicates if this tag +# applies to nodes, ways, or both. +# +# Tag: The tag +# +# DataType: The type of the column to be created. Normally "text" +# +# Flags: Flags that indicate what table the OSM object is moved into. +# +# There are 5 possible flags. These flags are used both to indicate if a column +# should be created, and if ways with the tag are assumed to be areas. The area +# assumptions can be overridden with an area=yes/no tag +# +# polygon - Create a column for this tag, and objects the tag with are areas +# +# linear - Create a column for this tag +# +# phstore - Don't create a column for this tag, but objects with the tag are areas +# +# delete - Drop this tag completely and don't create a column for it. This also +# prevents the tag from being added to hstore columns +# +# nocache - Deprecated and does nothing +# +# If an object has a tag that indicates it is an area or has area=yes/1, +# osm2pgsql will try to turn it into an area. If it succeeds, it places it in +# the polygon table. If it fails (e.g. not a closed way) it places it in the +# line table. +# +# Nodes are never placed into the polygon or line table and are always placed in +# the point table. +# +# Hstore +# ====== +# +# The options --hstore, --hstore-match-only, and --hstore-all interact with +# the .style file. +# +# With --hstore any tags without a column will be added to the hstore column. +# This will also cause all objects to be kept. +# +# With --hstore-match-only the behavior for tags is the same, but objects are +# only kept if they have a non-NULL value in one of the columns. +# +# With --hstore-all all tags are added to the hstore column unless they appear +# in the style file with a delete flag, causing duplication between the normal +# columns and the hstore column. +# +# Special database columns +# ======================== +# +# There are some special database columns that if present in the .style file +# will be populated by osm2pgsql. +# +# These are +# +# z_order - datatype int4 +# +# way_area - datatype real. The area of the way, in the units of the projection +# (e.g. square mercator meters). Only applies to areas +# +# osm_user - datatype text +# osm_uid - datatype integer +# osm_version - datatype integer +# osm_changeset - datatype integer +# osm_timestamp - datatype timestamptz(0). +# Used with the --extra-attributes option to include metadata in the database. +# If importing with both --hstore and --extra-attributes the meta-data will +# end up in the tags hstore column regardless of the style file. + +# OsmType Tag DataType Flags +node,way access text linear +node,way addr:housename text linear +node,way addr:housenumber text linear +node,way addr:interpolation text linear +node,way admin_level text linear +node,way aerialway text linear +node,way aeroway text polygon +node,way amenity text polygon +node,way area text polygon # hard coded support for area=1/yes => polygon is in osm2pgsql +node,way barrier text linear +node,way bicycle text linear +node,way brand text linear +node,way bridge text linear +node,way boundary text linear +node,way building text polygon +node capital text linear +node,way construction text linear +node,way covered text linear +node,way culvert text linear +node,way cutting text linear +node,way denomination text linear +node,way disused text linear +node ele text linear +node,way embankment text linear +node,way foot text linear +node,way generator:source text linear +node,way harbour text polygon +node,way highway text linear +node,way historic text polygon +node,way horse text linear +node,way intermittent text linear +node,way junction text linear +node,way landuse text polygon +node,way layer text linear +node,way leisure text polygon +node,way lock text linear +node,way man_made text polygon +node,way military text polygon +node,way motorcar text linear +node,way name text linear +node,way natural text polygon # natural=coastline tags are discarded by a hard coded rule in osm2pgsql +node,way office text polygon +node,way oneway text linear +node,way operator text linear +node,way place text polygon +node poi text linear +node,way population text linear +node,way power text polygon +node,way power_source text linear +node,way public_transport text polygon +node,way railway text linear +node,way ref text linear +node,way religion text linear +node,way route text linear +node,way service text linear +node,way shop text polygon +node,way sport text polygon +node,way surface text linear +node,way toll text linear +node,way tourism text polygon +node,way tower:type text linear +way tracktype text linear +node,way tunnel text linear +node,way water text polygon +node,way waterway text polygon +node,way wetland text polygon +node,way width text linear +node,way wood text linear +node,way z_order int4 linear # This is calculated during import +way way_area real linear # This is calculated during import + +# Area tags +# We don't make columns for these tags, but objects with them are areas. +# Mainly for use with hstore +way abandoned:aeroway text phstore +way abandoned:amenity text phstore +way abandoned:building text phstore +way abandoned:landuse text phstore +way abandoned:power text phstore +way area:highway text phstore + +# Deleted tags +# These are tags that are generally regarded as useless for most rendering. +# Most of them are from imports or intended as internal information for mappers +# Some of them are automatically deleted by editors. +# If you want some of them, perhaps for a debugging layer, just delete the lines. + +# These tags are used by mappers to keep track of data. +# They aren't very useful for rendering. +node,way note text delete +node,way note:* text delete +node,way source text delete +node,way source_ref text delete +node,way source:* text delete +node,way attribution text delete +node,way comment text delete +node,way fixme text delete + +# Tags generally dropped by editors, not otherwise covered +node,way created_by text delete +node,way odbl text delete +node,way odbl:note text delete +node,way SK53_bulk:load text delete + +# Lots of import tags +# TIGER (US) +node,way tiger:* text delete + +# NHD (US) +# NHD has been converted every way imaginable +node,way NHD:* text delete +node,way nhd:* text delete + +# GNIS (US) +node,way gnis:* text delete + +# Geobase (CA) +node,way geobase:* text delete +# NHN (CA) +node,way accuracy:meters text delete +node,way sub_sea:type text delete +node,way waterway:type text delete + +# KSJ2 (JA) +# See also note:ja and source_ref above +node,way KSJ2:* text delete +# Yahoo/ALPS (JA) +node,way yh:* text delete + +# osak (DK) +node,way osak:* text delete + +# kms (DK) +node,way kms:* text delete + +# ngbe (ES) +# See also note:es and source:file above +node,way ngbe:* text delete + +# naptan (UK) +node,way naptan:* text delete + +# Corine (CLC) (Europe) +node,way CLC:* text delete + +# misc +node,way 3dshapes:ggmodelk text delete +node,way AND_nosr_r text delete +node,way import text delete +node,way it:fvg:* text delete diff -Nru osm2pgsql-0.82.0/docs/analysis.md osm2pgsql-0.88.1/docs/analysis.md --- osm2pgsql-0.82.0/docs/analysis.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/analysis.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,81 @@ +# Geospatial analysis with osm2pgsql # + +An osm2pgsql database and PostGIS is well-suited for geospatial analysis using +OpenStreetMap data where topology is not a consideration. + +PostGIS provides an [extensive number of geometry functions](http://postgis.net/docs/manual-2.1/reference.html) +and a full description of how to perform analysis with them is beyond the +scope of a readme, but a simple example of finding the total road lengths by +classification for a municipality should help. + +To start with, we'll download the data for the region as an [extract from Geofabrik](http://download.geofabrik.de/) and import it with osm2pgsql. + + osm2pgsql --database gis --number-processes 4 --multi-geometry british-columbia-latest.osm.pbf + +``--multi-geometry`` (``-G``) is necessary for most analysis as it prevents +MULTIPOLYGONs from being split into multiple POLYGONs, a step that is +normally used to [increase rendering speed](http://paulnorman.ca/blog/2014/03/osm2pgsql-multipolygons) +but increases the complexity of analysis SQL. + +Loading should take about 10 minutes, depending on computer speed. Once this +is done we'll open a PostgreSQL terminal with ``psql -d gis``, although a GUI +like pgadmin or any standard tool could be used instead. + +To start, we'll create a partial index to speed up highway queries. + +```sql +CREATE INDEX planet_osm_line_highways_index ON planet_osm_line USING GiST (way) WHERE (highway IS NOT NULL); +``` + +We'll first find the ID of the polygon we want + +```sql +gis=# SELECT osm_id FROM planet_osm_polygon +WHERE boundary='administrative' AND admin_level='8' AND name='New Westminster'; + osm_id +---------- + -1377803 +``` + +The negative sign tells us that the geometry is from a relation, and checking +on [the OpenStreetMap site](https://www.openstreetmap.org/relation/1377803) +confirms which it is. + +We want to find all the roads in the city and get the length of the portion in +the city, sorted by road classification. Roads are in the ``planet_osm_line`` +table, not the ``planet_osm_roads`` table which is only has a subset of data +for low-zoom rendering. + +```sql +gis=# SELECT + round(SUM( + ST_Length(ST_Transform( + ST_Intersection(way, (SELECT way FROM planet_osm_polygon WHERE osm_id=-1377803)) + ,4326)::geography) + )) AS "distance (meters)", highway AS "highway type" + FROM planet_osm_line + WHERE highway IS NOT NULL + AND ST_Intersects(way, (SELECT way FROM planet_osm_polygon WHERE osm_id=-1377803)) + GROUP BY highway + ORDER BY "distance (meters)" DESC + LIMIT 10; + distance (meters) | highway type +-------------------+--------------- + 138122 | residential + 79519 | service + 51890 | footway + 25610 | tertiary + 23434 | secondary + 14900 | cycleway + 6468 | primary + 5217 | motorway + 4389 | motorway_link + 3728 | track +``` + +The ``ST_Transform(...,4326)::geography`` is necessary because the data was +imported in Mercator. This step could have been avoided by importing in a local +projection like a suitable UTM projection. + +More complicated analysises can be completed, but this simple example shows how +to use the tables and put conditions on the columns. \ No newline at end of file diff -Nru osm2pgsql-0.82.0/docs/export.md osm2pgsql-0.88.1/docs/export.md --- osm2pgsql-0.82.0/docs/export.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/export.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,12 @@ +# Exporting with osm2pgsql # + +Osm2pgsql can be used in combination with [ogr2ogr](http://www.gdal.org/ogr2ogr.html) and a [PostgreSQL data source](http://www.gdal.org/drv_pg.html). + +An example command to export to GeoJSON would be + + ogr2ogr -f "GeoJSON" roads.geojson -t_srs EPSG:4326 \ + PG:"dbname=gis" -s_srs EPSG:900913 \ + -sql "SELECT name,highway,oneway,toll,way FROM planet_osm_line WHERE highway IS NOT NULL" + +Care should be taken if exporting to shapefiles, as characters may be present +which cannot be represented in ISO-8859-1, the standard encoding for shapefiles. diff -Nru osm2pgsql-0.82.0/docs/gazetteer.md osm2pgsql-0.88.1/docs/gazetteer.md --- osm2pgsql-0.82.0/docs/gazetteer.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/gazetteer.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,7 @@ +# Gazetteer Backend # + +The gazetteer backend is designed for use with +[Nominatim](http://wiki.openstreetmap.org/wiki/Nominatim) +and will not generally be used outside that context. + +The tables are designed for a hiarchy of places. diff -Nru osm2pgsql-0.82.0/docs/lua.md osm2pgsql-0.88.1/docs/lua.md --- osm2pgsql-0.82.0/docs/lua.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/lua.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,57 @@ +# Lua tag transformations + +osm2pgsql supports [Lua](http://lua.org/) scripts to rewrite tags before they enter the database. + +This allows you to unify disparate tagging (for example, `highway=path; foot=yes` and `highway=footway`) and perform complex queries, potentially more efficiently than writing them as rules in your Mapnik or other stylesheet. + +## How to + +Pass a Lua script to osm2pgsql using the command line switch `--tag-transform-script`: + + osm2pgsql -S your.style --tag-transform-script your.lua --hstore-all extract.osm.pbf + +This Lua script needs to implement the following functions: + + function filter_tags_node(tags, num_tags) + return filter, tags + + function filter_tags_way(tags, num_tags) + return filter, tags, polygon, roads + + function filter_basic_tags_rel(tags, num_tags) + return filter, tags + +These take a set of tags as a Lua key-value table, and an integer which is the number of tags supplied. + +The first return value is `filter`, a flag which you should set to `1` if the way/node/relation should be filtered out and not added to the database, `0` otherwise. (They will still end up in the slim mode tables, but not in the rendering tables) + +The second return value is `tags`, a transformed (or unchanged) set of tags. + +`filter_tags_way` returns two additional flags. `poly` should be `1` if the way should be treated as a polygon, `0` as a line. `roads` should be `1` if the way should be added to the planet_osm_roads table, `0` otherwise. + + function filter_tags_relation_member(tags, member_tags, + roles, num_members) + return filter, tags, member_superseded, boundary, + polygon, roads + +The function filter_tags_relation_member is more complex and can handle more advanced relation tagging, such as multipolygons that take their tags from the member ways. + +This function is called with the tags from the relation; an set of tags for each of the member ways (member relations and nodes are ignored); the set of roles for each of the member ways; and the number of members. The tag and role sets are both arrays (indexed tables) of hashes (tables). + +As usual, it should return a filter flag, and a transformed set of tags to be applied to the relation in later processing. + +The third return value, `member_superseded`, is a flag set to `1` if the way has now been dealt with (e.g. outer ways in multipolygon relations, which are superseded by the multipolygon geometry), `0` if it needs to have its own entry in the database (e.g. tagged inner ways). + +The fourth and fifth return values, `boundary` and `polygon`, are flags that specify if the relation should be processed as a line, a polygon, or both (e.g. administrative boundaries). + +The final return value, `roads`, is `1` if the geometry should be added to the `planet_osm_roads` table. + +There is a sample tag transform lua script in the repository as an example, which (nearly) replicates current processing and can be used as a template for one's own scripts. + +## In practice + +There is inevitably a performance hit with any extra processing. The sample Lua tag transformation is a little slower than the C-based default. However, extensive Lua pre-processing may save you further processing in your Mapnik (or other) stylesheet. + +Test your Lua script with small excerpts before applying it to a whole country or even the planet. + +Where possible, add new tags, don't replace existing ones; otherwise you will be faced with a reimport if you decide to change your transformation. diff -Nru osm2pgsql-0.82.0/docs/migrations.md osm2pgsql-0.88.1/docs/migrations.md --- osm2pgsql-0.82.0/docs/migrations.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/migrations.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,37 @@ +# Migrations between versions # + +Some osm2pgsql changes have slightly changed the database schema it expects. If +updating an old database, a migration may be needed. The migrations here assume +the default `planet_osm` prefix. + +## 0.88.0 z_order changes ## + +0.88.0 z_order logic was changed, requuiring an increase in z_order values. To +migrate to the new range of values, run + +```sql +UPDATE planet_osm_line SET z_order = z_order * 10; +UPDATE planet_osm_roads SET z_order = z_order * 10; +``` + +This will not apply the new logic, but will get the existing z_orders in the right +group of 100 for the new logic. + +If not using osm2pgsql z_orders, this change may be ignored. + +## 0.87.0 pending removal ## + +0.87.0 moved the in-database tracking of pending ways and relations to +in-memory, for an increase in speed. This requires removal of the pending +column and a partial index associated with it. + +```sql +ALTER TABLE planet_osm_ways DROP COLUMN pending; +ALTER TABLE planet_osm_rels DROP COLUMN pending; +``` + +## 32 bit to 64 bit ID migration ## + +Old databases may have been imported with 32 bit node IDs, while current OSM +data requires 64 bit IDs. A database this old should not be migrated, but +reloaded. To migrate, the type of ID columns needs to be changed to `bigint`. diff -Nru osm2pgsql-0.82.0/docs/multi.md osm2pgsql-0.88.1/docs/multi.md --- osm2pgsql-0.82.0/docs/multi.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/multi.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,61 @@ +# Multi Backend # + +The multi backend is designed for custom table structures as an alternative +to the standard [pgsql](pgsql.md) backend tables. It is intended to allow +the configuration of a custom set of tables with hopefully fewer rows and fewer +columns. This can be beneficial to queries in which some context (eg. zoom level) +could limit the number of tables that need to be queried. Addtionaly it would +allow more tables to be queried in parallel. + +## Database Layout ## +It connects to a PostgreSQL database and stores the data in one or more tables. +Each table is configured in a way similar to that of the `pgsql` backend. +That is essentially why it was named `multi` because it's basically multiple +`pgsql` backends each with its own set of options and only a single table. + +## Table Configuration ## +As sample configuration may resemble the following: + + [ + { + "name": "building", + "type": "polygon", + "tagtransform": "building.lua", + "tagtransform-node-function": "nodes_proc", + "tagtransform-way-function": "ways_proc", + "tagtransform-relation-function": "rels_proc", + "tagtransform-relation-member-function": "rel_members_proc", + "tags": [ + {"name": "building", "type": "text"}, + {"name": "shop", "type": "text"}, + {"name": "amenity", "type": "text"} + ] + }, + ... + ] + +Note that each table has a `name` and can target a single type of geometry +by setting the `type` to one of `point`, `line` or `polygon`. `tagtransform` +is used to set the name of the lua script to be used for custom tag processing. +Within the lua script you may define several methods that will be called +when processing various tags, these can be named via +`tagtransform-node-function`, `tagtransform-way-function`, +`tagtransform-relation-function`, and `tagtransform-relation-member-function`. +As with the normal top level options within osm2pgsql you can specify any of the +following: `tablespace-index`, `tablespace-data`, `enable-hstore`, +`enable-hstore-index`, `enable-multi`, `hstore-match-only`. Hstore colum names +may be specified via an array of strings named `hstores`. Finally standard columns +may be specified via an array of objects named `tags` with each object containing +a `name` and a postgres `type`. Note you may also set `flags` on each tag as with +the standard osm2pgsql style file. `flags` is formated exactly as in the style file +as a string of flag names seprated by commas. + +## Example ## +An example based on the above is in [multi.lua](../multi.lua) and +[multi.style.json](../multi.style.json). It creates three tables, one for bus +stops, one for buildings, and one for highways. Some Lua processing is done to +unify tagging values. + +## Importing ## + +See: [Importing](pgsql.md#importing). diff -Nru osm2pgsql-0.82.0/docs/nodecachefilereader.1 osm2pgsql-0.88.1/docs/nodecachefilereader.1 --- osm2pgsql-0.82.0/docs/nodecachefilereader.1 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/docs/nodecachefilereader.1 2015-08-15 05:28:11.000000000 +0000 @@ -18,7 +18,7 @@ command. .PP .B nodecachefilereader -allows you to inspect and test osm2pgsql's custome node database. +allows you to inspect and test osm2pgsql's custom node database. .PP .SH OPTIONS If only the filename of the node cache is given, nodecachefilereader @@ -34,4 +34,4 @@ .SH AUTHOR nodecachefilereader was written by Kai Krueger and other OpenStreetMap project members. -.PP \ No newline at end of file +.PP diff -Nru osm2pgsql-0.82.0/docs/osm2pgsql.1 osm2pgsql-0.88.1/docs/osm2pgsql.1 --- osm2pgsql-0.82.0/docs/osm2pgsql.1 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/docs/osm2pgsql.1 2015-08-15 05:28:11.000000000 +0000 @@ -36,7 +36,7 @@ .PP .SH OPTIONS These programs follow the usual GNU command line syntax, with long -options starting with two dashes (`-'). +options starting with two dashes (`\-'). A summary of options is included below. .TP \fB\-a\fR|\-\-append @@ -44,9 +44,9 @@ existing data. .TP \fB\-b\fR|\-\-bbox -Apply a bounding box filter on the imported data +Apply a bounding box filter on the imported data. Must be specified as: minlon,minlat,maxlon,maxlat -e.g. \fB\-\-bbox\fR \fB\-0\fR.5,51.25,0.5,51.75 +e.g. \fB\-\-bbox\fR \fB\-0.5,51.25,0.5,51.75\fR .TP \fB\-c\fR|\-\-create Remove existing data from the database. This is the @@ -58,49 +58,46 @@ .TP \fB\-i\fR|\-\-tablespace\-index tablespacename Store all indices in a separate PostgreSQL tablespace named by this parameter. -This allows to e.g. store the indices on faster storage like SSDs +This allows one to e.g. store the indices on faster storage like SSDs. .TP \fB\ \fR\-\-tablespace\-main\-data tablespacename -Store the data tables (non slim) in the given tablespace +Store the data tables (non slim) in the given tablespace. .TP \fB\ \fR\-\-tablespace\-main\-index tablespacename -Store the indices of the main tables (non slim) in the given tablespace +Store the indices of the main tables (non slim) in the given tablespace. .TP \fB\ \fR\-\-tablespace\-slim\-data tablespacename -Store the slim mode tables in the given tablespace +Store the slim mode tables in the given tablespace. .TP \fB\ \fR\-\-tablespace\-slim\-index tablespacename -Store the indices of the slim mode tables in the given tablespace +Store the indices of the slim mode tables in the given tablespace. .TP \fB\-l\fR|\-\-latlong Store data in degrees of latitude & longitude. .TP \fB\-m\fR|\-\-merc -Store data in proper spherical Mercator (the default) -.TP -\fB\-M\fR|\-\-oldmerc -Store data in the legacy OSM Mercator format +Store data in proper spherical Mercator (the default). .TP \fB\-E\fR|\-\-proj num Use projection EPSG:num .TP \fB\-u\fR|\-\-utf8\-sanitize -Repair bad UTF8 input data (present in planet +Repair bad UTF\-8 input data (present in planet dumps prior to August 2007). Adds about 10% overhead. .TP \fB\-p\fR|\-\-prefix prefix_string -Prefix for table names (default planet_osm) +Prefix for table names (default: planet_osm). .TP \fB\-r\fR|\-\-input\-reader format Select input format reader. Available choices are \fBlibxml2\fR -(default) and \fBprimitive\fR for OSM XML format files, \fBo5m\fR for o5m formatted file -and \fBpbf\fR for OSM PBF binary format (may not be available on all platforms) +(default) for OSM XML format files, \fBo5m\fR for o5m formatted file +and \fBpbf\fR for OSM PBF binary format (may not be available on all platforms). .TP \fB\-s\fR|\-\-slim Store temporary data in the database. Without this mode, all temporary data is stored in -RAM and if you do not have enough the import will not succeed successfully. With slim mode, +RAM and if you do not have enough the import will not work successfully. With slim mode, you should be able to import the data even on a system with limited RAM, although if you -do no have enough RAM to cache at least all of the nodes, the time to import the data +do not have enough RAM to cache at least all of the nodes, the time to import the data will likely be greatly increased. .TP \fB\ \fR\-\-drop @@ -109,7 +106,7 @@ size, if not slightly bigger than the main tables. It does not, however, reduce the maximum spike of disk usage during import. It can furthermore increase the import speed, as no indices need to be created for the slim mode tables, which (depending on hardware) -can nearly half import time. Slim mode tables however have to be persistent if you want +can nearly halve import time. Slim mode tables however have to be persistent if you want to be able to update your database, as these tables are needed for diff processing. .TP \fB\-S\fR|\-\-style /path/to/style @@ -119,11 +116,11 @@ \fB\-C\fR|\-\-cache num Only for slim mode: Use up to num many MB of RAM for caching nodes. Giving osm2pgsql sufficient cache to store all imported nodes typically greatly increases the speed of the import. Each cached node -requires 8 bytes of cache, plus about 10% - 30% overhead. For a current OSM full planet import with -its ~ 1.9 billion nodes, a good value would be 17000 if you have enough RAM. If you don't have enough +requires 8 bytes of cache, plus about 10% \- 30% overhead. For a current OSM full planet import with +its ~ 3 billion nodes, a good value would be 27000 if you have enough RAM. If you don't have enough RAM, it is likely beneficial to give osm2pgsql close to the full available amount of RAM. Defaults to 800. .TP -\fB\ \fR\-\-cache-strategy strategy +\fB\ \fR\-\-cache\-strategy strategy There are a number of different modes in which osm2pgsql can organize its node cache in RAM. These are optimized for different assumptions of the data and the hardware resources available. Currently available strategies are @@ -150,16 +147,16 @@ \fB\-P\fR|\-\-port num Database server port. .TP -\fB\-e\fR|\-\-expire-tiles [min_zoom-]max-zoom +\fB\-e\fR|\-\-expire\-tiles [min_zoom\-]max\-zoom Create a tile expiry list. .TP -\fB\-o\fR|\-\-expire-output /path/to/expire.list +\fB\-o\fR|\-\-expire\-output /path/to/expire.list Output file name for expired tiles list. .TP \fB\-o\fR|\-\-output -Specifies the output back-end or database schema to use. Currently +Specifies the output back\-end or database schema to use. Currently osm2pgsql supports \fBpgsql\fR, \fBgazetteer\fR and \fBnull\fR. \fBpgsql\fR is -the default output back-end / schema and is optimized for rendering with Mapnik. +the default output back\-end / schema and is optimized for rendering with Mapnik. \fBgazetteer\fR is a db schema optimized for geocoding and is used by Nominatim. \fBnull\fR does not write any output and is only useful for testing. .TP @@ -169,34 +166,34 @@ Note: this option also requires additional entries in your style file. .TP \fB\-k\fR|\-\-hstore -Add tags without column to an additional hstore (key/value) column to PostgreSQL tables +Add tags without column to an additional hstore (key/value) column to PostgreSQL tables. .TP \fB\-j\fR|\-\-hstore\-all -Add all tags to an additional history (key/value) column in PostgreSQL tables +Add all tags to an additional hstore (key/value) column in PostgreSQL tables. .TP \fB\-z\fR|\-\-hstore\-column key_name -Add an additional history (key/value) column containing all tags -that start with the specified string, egg --hstore-column "name:" will +Add an additional hstore (key/value) column containing all tags +that start with the specified string, eg \-\-hstore\-column "name:" will produce an extra hstore column that contains all name:xx tags .TP \fB\ \fR\-\-hstore\-match\-only Only keep objects that have a value in one of the columns -(normal action with --hstore is to keep all objects) +(normal action with \-\-hstore is to keep all objects). .TP -\fB\ \fR\-\-hstore-add-index +\fB\ \fR\-\-hstore\-add\-index Create indices for the hstore columns during import. .TP \fB\-G\fR|\-\-melts\-geometry -Normally osm2pgsql splits multi-part geometries into separate database rows per part. +Normally osm2pgsql splits multi\-part geometries into separate database rows per part. A single OSM id can therefore have several rows. With this option, PostgreSQL instead -generates multi-geometry features in the PostgreSQL tables. +generates multi\-geometry features in the PostgreSQL tables. .TP -\fB\-K\fR|\-\-keep-coastlines +\fB\-K\fR|\-\-keep\-coastlines Keep coastline data rather than filtering it out. By default natural=coastline tagged data will be discarded based on the assumption that post-processed Coastline Checker shape files will be used. .TP -\fB\ \fR\-\-exclude-invalid-polygon +\fB\ \fR\-\-exclude\-invalid\-polygon OpenStreetMap data is defined in terms of nodes, ways and relations and not in terms of actual geometric features. Osm2pgsql therefore tries to build postgis geometries out of this data representation. However not all ways and relations @@ -207,33 +204,33 @@ .TP \fB\ \fR\-\-unlogged Use postgresql's unlogged tables for storing data. This requires PostgreSQL 9.1 -or above. Data written to unlogged tables is not written to PostgreSQL's write-ahead log, +or above. Data written to unlogged tables is not written to PostgreSQL's write\-ahead log, which makes them considerably faster than ordinary tables. However, they are not -crash-safe: an unlogged table is automatically truncated after a crash or unclean shutdown. +crash\-safe: an unlogged table is automatically truncated after a crash or unclean shutdown. .TP \fB\ \fR\-\-number\-processes num Specifies the number of parallel processes used for certain operations. If disks are fast enough e.g. if you have an SSD, then this can greatly increase speed of -the "going over pending ways" and "going over pending relations" stages on a multi-core -server. +the "going over pending ways" and "going over pending relations" stages on a multi\-core +server. .TP -\fB\-I\fR|\-\-disable-parallel-indexing +\fB\-I\fR|\-\-disable\-parallel\-indexing By default osm2pgsql initiates the index building on all tables in parallel to increase performance. This can be disadvantages on slow disks, or if you don't have -enough ram for PostgreSQL to perform up to 7 parallel index building processes -(e.g. because maintenance_work_mem is set high) +enough RAM for PostgreSQL to perform up to 7 parallel index building processes +(e.g. because maintenance_work_mem is set high). .TP -\fB\ \fR\-\-flat-nodes /path/to/nodes.cache -The flat-nodes mode is a separate method to store slim mode node information on disk. +\fB\ \fR\-\-flat\-nodes /path/to/nodes.cache +The flat\-nodes mode is a separate method to store slim mode node information on disk. Instead of storing this information in the main PostgreSQL database, this mode creates its own separate custom database to store the information. As this custom database has application level knowledge about the data to store and is not general purpose, -it can store the data much more efficient. Storing the node information for the full +it can store the data much more efficiently. Storing the node information for the full planet requires about 100GB in PostgreSQL, the same data is stored in only ~16GB using -the flat-nodes mode. This can also increase the speed of applying diff files. This option -activates the flat-nodes mode and specifies the location of the database file. It is a +the flat\-nodes mode. This can also increase the speed of applying diff files. This option +activates the flat\-nodes mode and specifies the location of the database file. It is a single large > 16GB file. This mode is only recommended for full planet imports -as it doesn't work well with small extracts. The default is disabled +as it doesn't work well with small extracts. The default is disabled. .TP \fB\-h\fR|\-\-help Help information. @@ -244,11 +241,11 @@ Verbose output. .PP .SH SUPPORTED PROJECTIONS -Latlong (-l) SRS: 4326 (none) -.br -WGS84 Mercator ( ) SRS: 3395 +proj=merc +datum=WGS84 +k=1.0 +units=m +over +no_defs +Latlong (\-l) SRS: 4326 (none) +.br +Spherical Mercator (\-m) SRS:900913 +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over .br -Spherical Mercator (-m) SRS:900913 +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs +over +EPSG-defined (\-E) SRS: +init=epsg:(as given in parameter) .PP .SH SEE ALSO .BR proj (1), diff -Nru osm2pgsql-0.82.0/docs/pgsql.md osm2pgsql-0.88.1/docs/pgsql.md --- osm2pgsql-0.82.0/docs/pgsql.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/pgsql.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,54 @@ +# Pgsql Backend # + +The pgsql backend is designed for rendering OpenStreetMap data, principally +with Mapnik, but is also useful for [analysis](docs/analysis.md) and +[exporting](docs/exporting.md) to other formats. + +## Database Layout ## +It connects to a PostgreSQL database and stores the data in four tables + +* ``planet_osm_point`` +* ``planet_osm_line`` +* ``planet_osm_roads`` +* ``planet_osm_polygon`` + +planet_osm_roads contains the data from other tables, but has tags selected +for low-zoom rendering. It does not only contain roads. + +The default prefix ``planet_osm`` can be changed with the ``--prefix`` option. + +If you are using ``--slim`` mode, it will create the following additional 3 +tables which are used by the pgsql middle layer, not the backend: + +* ``planet_osm_nodes`` +* ``planet_osm_ways`` +* ``planet_osm_rels`` + +With the ``--flat-nodes`` option, the ``planet_osm_nodes`` information is +instead stored in a binary file. + +## Importing ## + +1. Runs a parser on the input file and processes the nodes, ways and relations. + +2. If a node has a tag declared in the style file then it is added to + ``planet_osm_point``. Regardless of tags, its position is stored by the + middle layer. + +3. If there are tags on a way in the style file as linear but without polygon + tags, they are written into the lines and, depending on tags, roads tables. + + They are also stored by the middle layer. + +4. Ways without tags or with polygon tags are stored as "pending" in the + middle layer. + +5. Relations are parsed. In this stage, "new-style" multipolygon and boundary + relations are turned into polygons. Route relations are turned into + linestrings. + +6. "Pending" ways are processed, and they are either added as just the way, or + if a member of a multipolygon relation, they processed as multipolygons. + +7. Indexes are built. This may take substantial time, particularly for the + middle layer indexes created in non-slim mode. diff -Nru osm2pgsql-0.82.0/docs/usage.md osm2pgsql-0.88.1/docs/usage.md --- osm2pgsql-0.82.0/docs/usage.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/docs/usage.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,149 @@ +# Command-line usage # + +Osm2pgsql has one program, the executable itself, which has **43** command line +options. A full list of options can be obtained with ``osm2pgsql -h -v``. This +document provides an overview of options, and more importantly, why you might +use them. + +## Overall options + +* ``--append`` or ``--create`` specify if osm2pgsql is conducting a new import + or adding to an existing one. ``--slim`` is required with ``--append``. + +* ``--input-reader`` specifies the parser if the filetype can't be + automatically detected for some reason. + +* ``--output`` specifies if the output backend is the default + [`pgsql`](pgsql.md), the [`gazetteer`](gazetteer.md) output used by Nominatim, + the new [`multi`](multi.md) backend which allows more customization of tables, + or `null`, which emits no output to the backend. + + `null` will create slim tables if ``--slim`` is also used. + +## Performance + +Performance is heavily influenced by other options, but there are some options +that only impact performance. + +* ``--cache`` specifies how much memory to allocate for caching information. In + ``--slim`` mode, this is just node positions while in non-slim it has to + store information about ways and relations too. The maximum RAM it is useful + to set this to in slim mode is 8 bytes * number of nodes / efficiency, where + efficiency ranges from 50% on small extracts to 80% for a planet. + +* ``--number-processes`` sets the number of processes to use. This should + typically be set to the number of CPU threads, but gains in speed are minimal + past 8 threads. + +* ``--disable-parallel-indexing`` disables the clustering and indexing of all + tables in parallel. This reduces disk and ram requirements during the import, + but causes the last stages to take significantly longer. + +* ``--cache-strategy`` sets the cache strategy to use. The defaults are fine + here, and optimizied uses less RAM than the other options. + +## Database options ## + +osm2pgsql supports standard options for how to connect to PostgreSQL. If left +unset, it will attempt to connect to the ``gis`` database using a unix socket. +Most usage only requires setting ``--database``. + +``--tablespace`` options allow the location of main and slim tables and indexes +to be set to different tablespaces independently, typically on machines with +multiple drive arrays where one is not large enough for all of the database. + +``--flat-nodes`` specifies that instead of a table in PostgreSQL, a binary +file is used as a database of node locations. This should only be used on full +planet imports or very large extracts (e.g. Europe) but in those situations +offers significant space savings and speed increases, particularly on +mechanical drives. The file takes approximately 8 bytes * maximum node ID, or +about 23 GiB, regardless of the size of the extract. + +``--unlogged`` specifies to use unlogged tables which are dropped from the +database if the database server ever crashes, but are faster to import. + +``--prefix`` specifies the prefix for tables + +## Middle-layer options ## + +* ``--slim`` causes the middle layer to store node and way information in + database rather than in memory. It is required for updates and for large + extracts or the entire planet which will not fit in RAM. + +* ``--drop`` discards the slim tables when they are no longer needed in the + import, significantly reducing disk requirements and saving the time of + building slim table indexes. A ``--slim --drop`` import is generally the + fastest way to import the planet if updates are not required. + +## Output columns options ## + +### Column options + +* ``--extra-attributes`` creates psudo-tags with OSM meta-data like user, + last edited, and changeset. These also need to be added to the style file. + +* ``--style`` specifies the location of the style file. This defines what + columns are created, what tags denote areas, and what tags can be ignored. + The [default.style](../default.style) contains more documentation on this + file. + +* ``--tag-transform-script`` sets a [Lua tag transform](lua.md) to use in + place of the built-in C tag transform. + +### Hstore + +Hstore is a [PostgreSQL data type](http://www.postgresql.org/docs/9.3/static/hstore.html) +that allows storing arbitrary key-value pairs. It needs to be installed on +the database with ``CREATE EXTENSION hstore;`` + +osm2pgsql has five hstore options + +* ``--hstore`` or ``-k`` adds any tags not already in a conventional column to + a hstore column. With the standard stylesheet this would result in tags like + highway appearing in a conventional column while tags not in the style like + ``name:en`` or ``lanes:forward`` would appear only in the hstore column. + +* ``--hstore-all`` or ``-j`` adds all tags to a hstore column, even if they're + already stored in a conventional column. With the standard stylesheet this + would result in tags like highway appearing in conventional column and the + hstore column while tags not in the style like ``name:en`` or + ``lanes:forward`` would appear only in the hstore column. + +* ``--hstore-column`` or ``-z``, which adds an additional column for tags + starting with a specified string, e.g. ``--hstore-column 'name:'`` produces + a hstore column that contains all ``name:xx`` tags + +* ``--hstore-match-only`` modifies the above options and prevents objects from + being added if they only have tags in the hstore column and no conventional + tags. + +* ``--hstore-add-index`` adds a GIN index to the hstore columns. This can + speed up arbitrary queries, but for most purposes partial indexes will be + faster. + +Either ``--hstore`` or ``--hstore-all`` when combined with ``--hstore-match-only`` +should give the same rows as no hstore, just with the additional hstore column. + +Hstore is used to give more flexability to use additional tags without +reimporting the database, at the cost of a +[less speed and more space.](http://paulnorman.ca/blog/2014/03/osm2pgsql-and-hstore/) + +## Projection options + +* ``--latlong``, ``--merc``, or ``--proj`` are used to specify the projection + used for importing. The default, ``--merc`` is typically used for rendering, + while ``--latlong`` can offer advantages for analysis. Most stylesheets + assume ``--merc`` has been used. + +## Output data options + +* ``--multi-geometry`` skips an optimization for rendering where PostGIS + MULTIPOLYGONs are split into multiple POLYGONs. ``--multi-geometry`` can be + used to [avoid some labeling issues at the cost of speed](http://paulnorman.ca/blog/2014/03/osm2pgsql-multipolygons/). + It is also typically required for [analysis](analysis.md). + +* ``--keep-coastlines`` disables a hard-coded rule that would otherwise + discard ``natural=coastline`` ways. + +* ``--exclude-invalid-polygon`` prevents osm2pgsql from attempting to form + valid polygons from invalid ones and just rejects the invalid ones. diff -Nru osm2pgsql-0.82.0/empty.style osm2pgsql-0.88.1/empty.style --- osm2pgsql-0.82.0/empty.style 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/empty.style 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,107 @@ +# This osm2pgsql style file is one that will generate no columns from tags +# It is designed as a starting point for you to develop your own, or for +# use where all OSM tags are in hstore. + +# See default.style for documentation on all the flags + +# OsmType Tag Type Flags +# Insert your own columns here, or change phstore to polygon below +way abandoned:aeroway text phstore +way abandoned:amenity text phstore +way abandoned:building text phstore +way abandoned:landuse text phstore +way abandoned:power text phstore +way area:highway text phstore +node,way aeroway text phstore +node,way amenity text phstore +node,way building text phstore +way building:part text phstore +node,way harbour text phstore +node,way historic text phstore +node,way landuse text phstore +node,way leisure text phstore +node,way man_made text phstore +node,way military text phstore +node,way natural text phstore +node,way office text phstore +node,way place text phstore +node,way power text phstore +node,way public_transport text phstore +node,way shop text phstore +node,way sport text phstore +node,way tourism text phstore +node,way water text phstore +node,way waterway text phstore +node,way wetland text phstore +node,way z_order int4 linear # This is calculated during import +way way_area real linear # This is calculated during import + +# Deleted tags +# These are tags that are generally regarded as useless for most rendering. +# Most of them are from imports or intended as internal information for mappers +# Some of them are automatically deleted by editors. +# If you want some of them, perhaps for a debugging layer, just delete the lines. + +# These tags are used by mappers to keep track of data. +# They aren't very useful for rendering. +node,way note text delete +node,way note:* text delete +node,way source text delete +node,way source_ref text delete +node,way source:* text delete +node,way attribution text delete +node,way comment text delete +node,way fixme text delete + +# Tags generally dropped by editors, not otherwise covered +node,way created_by text delete +node,way odbl text delete +node,way odbl:note text delete +node,way SK53_bulk:load text delete + +# Lots of import tags +# TIGER (US) +node,way tiger:* text delete + +# NHD (US) +# NHD has been converted every way imaginable +node,way NHD:* text delete +node,way nhd:* text delete + +# GNIS (US) +node,way gnis:* text delete + +# Geobase (CA) +node,way geobase:* text delete +# NHN (CA) +node,way accuracy:meters text delete +node,way sub_sea:type text delete +node,way waterway:type text delete + +# KSJ2 (JA) +# See also note:ja and source_ref above +node,way KSJ2:* text delete +# Yahoo/ALPS (JA) +node,way yh:* text delete + +# osak (DK) +node,way osak:* text delete + +# kms (DK) +node,way kms:* text delete + +# ngbe (ES) +# See also note:es and source:file above +node,way ngbe:* text delete + +# naptan (UK) +node,way naptan:* text delete + +# Corine (CLC) (Europe) +node,way CLC:* text delete + +# misc +node,way 3dshapes:ggmodelk text delete +node,way AND_nosr_r text delete +node,way import text delete +node,way it:fvg:* text delete diff -Nru osm2pgsql-0.82.0/expire-tiles.c osm2pgsql-0.88.1/expire-tiles.c --- osm2pgsql-0.82.0/expire-tiles.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/expire-tiles.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,457 +0,0 @@ -/* - * Dirty tile list generation - * - * Steve Hill - * - * Please refer to the OpenPisteMap expire_tiles.py script for a demonstration - * of how to make use of the output: - * https://subversion.nexusuk.org/trac/browser/openpistemap/trunk/scripts/expire_tiles.py - */ - -#include -#include -#include -#include -#include -#include "expire-tiles.h" -#include "output.h" -#include "pgsql.h" -#include "build_geometry.h" -#include "reprojection.h" - -#define EARTH_CIRCUMFERENCE 40075016.68 -#define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2) -#define TILE_EXPIRY_LEEWAY 0.1 /* How many tiles worth of space to leave either side of a changed feature */ -#define EXPIRE_TILES_MAX_BBOX 20000 /* Maximum width or height of a bounding box (metres) */ - -struct tile { - int complete[2][2]; /* Flags */ - struct tile * subtiles[2][2]; -}; - -int map_width; /* not "static" since used in reprojection.c! */ -static double tile_width; -static const struct output_options * Options; -static struct tile * dirty = NULL; -static int outcount; - -/* - * We store the dirty tiles in an in-memory tree during runtime - * and dump them out to a file at the end. This allows us to easilly drop - * duplicate tiles from the output. - * - * This data structure consists of a node, representing a tile at zoom level 0, - * which contains 4 pointers to nodes representing each of the child tiles at - * zoom level 1, and so on down the the zoom level specified in - * Options->expire_tiles_zoom. - * - * The memory allowed to this structure is not capped, but daily deltas - * generally produce a few hundred thousand expired tiles at zoom level 17, - * which are easilly accommodated. - */ - -static int calc_complete(struct tile * tile) { - int c; - - c = tile->complete[0][0]; - c += tile->complete[0][1]; - c += tile->complete[1][0]; - c += tile->complete[1][1]; - return c; -} - -static void destroy_tree(struct tile * tree) { - if (! tree) return; - if (tree->subtiles[0][0]) destroy_tree(tree->subtiles[0][0]); - if (tree->subtiles[0][1]) destroy_tree(tree->subtiles[0][1]); - if (tree->subtiles[1][0]) destroy_tree(tree->subtiles[1][0]); - if (tree->subtiles[1][1]) destroy_tree(tree->subtiles[1][1]); - free(tree); -} - -/* - * Mark a tile as dirty. - * Returns the number of subtiles which have all their children marked as dirty. - */ -static int _mark_tile(struct tile ** tree, int x, int y, int zoom, int this_zoom) { - int zoom_diff = zoom - this_zoom; - int rel_x; - int rel_y; - int complete; - - if (! *tree) *tree = calloc(1, sizeof(**tree)); - zoom_diff = (zoom - this_zoom) - 1; - rel_x = (x >> zoom_diff) & 1; - rel_y = (y >> zoom_diff) & 1; - if (! (*tree)->complete[rel_x][rel_y]) { - if (zoom_diff <= 0) { - (*tree)->complete[rel_x][rel_y] = 1; - } else { - complete = _mark_tile(&((*tree)->subtiles[rel_x][rel_y]), x, y, zoom, this_zoom + 1); - if (complete >= 4) { - (*tree)->complete[rel_x][rel_y] = 1; - /* We can destroy the subtree to save memory now all the children are dirty */ - destroy_tree((*tree)->subtiles[rel_x][rel_y]); - (*tree)->subtiles[rel_x][rel_y] = NULL; - } - } - } - return calc_complete(*tree); -} - -/* - * Mark a tile as dirty. - * Returns the number of subtiles which have all their children marked as dirty. - */ -static int mark_tile(struct tile ** tree_head, int x, int y, int zoom) { - return _mark_tile(tree_head, x, y, zoom, 0); -} - -static void output_dirty_tile(FILE * outfile, int x, int y, int zoom, int min_zoom) { - int y_min; - int x_iter; - int y_iter; - int x_max; - int y_max; - int out_zoom; - int zoom_diff; - - if (zoom > min_zoom) out_zoom = zoom; - else out_zoom = min_zoom; - zoom_diff = out_zoom - zoom; - y_min = y << zoom_diff; - x_max = (x + 1) << zoom_diff; - y_max = (y + 1) << zoom_diff; - for (x_iter = x << zoom_diff; x_iter < x_max; x_iter++) { - for (y_iter = y_min; y_iter < y_max; y_iter++) { - outcount++; - if ((outcount <= 1) || (! (outcount % 1000))) { - fprintf(stderr, "\rWriting dirty tile list (%iK)", outcount / 1000); - fflush(stderr); - } - fprintf(outfile, "%i/%i/%i\n", out_zoom, x_iter, y_iter); - } - } -} - -static void _output_and_destroy_tree(FILE * outfile, struct tile * tree, int x, int y, int this_zoom, int min_zoom) { - int sub_x = x << 1; - int sub_y = y << 1; - FILE * ofile; - - if (! tree) return; - - ofile = outfile; - if ((tree->complete[0][0]) && outfile) { - output_dirty_tile(outfile, sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom); - ofile = NULL; - } - if (tree->subtiles[0][0]) _output_and_destroy_tree(ofile, tree->subtiles[0][0], sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom); - - ofile = outfile; - if ((tree->complete[0][1]) && outfile) { - output_dirty_tile(outfile, sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom); - ofile = NULL; - } - if (tree->subtiles[0][1]) _output_and_destroy_tree(ofile, tree->subtiles[0][1], sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom); - - ofile = outfile; - if ((tree->complete[1][0]) && outfile) { - output_dirty_tile(outfile, sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom); - ofile = NULL; - } - if (tree->subtiles[1][0]) _output_and_destroy_tree(ofile, tree->subtiles[1][0], sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom); - - ofile = outfile; - if ((tree->complete[1][1]) && outfile) { - output_dirty_tile(outfile, sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom); - ofile = NULL; - } - if (tree->subtiles[1][1]) _output_and_destroy_tree(ofile, tree->subtiles[1][1], sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom); - - free(tree); -} - -static void output_and_destroy_tree(FILE * outfile, struct tile * tree) { - _output_and_destroy_tree(outfile, tree, 0, 0, 0, Options->expire_tiles_zoom_min); -} - -void expire_tiles_stop(void) { - FILE * outfile; - - if (Options->expire_tiles_zoom < 0) return; - outcount = 0; - if ((outfile = fopen(Options->expire_tiles_filename, "a"))) { - output_and_destroy_tree(outfile, dirty); - fclose(outfile); - } else { - fprintf(stderr, "Failed to open expired tiles file (%s). Tile expiry list will not be written!\n", strerror(errno)); - } - dirty = NULL; -} - -void expire_tiles_init(const struct output_options *options) { - Options = options; - if (Options->expire_tiles_zoom < 0) return; - map_width = 1 << Options->expire_tiles_zoom; - tile_width = EARTH_CIRCUMFERENCE / map_width; -} - -static void expire_tile(int x, int y) { - mark_tile(&dirty, x, y, Options->expire_tiles_zoom); -} - -static int normalise_tile_x_coord(int x) { - x %= map_width; - if (x < 0) x = (map_width - x) + 1; - return x; -} - -/* - * Expire tiles that a line crosses - */ -static void expire_tiles_from_line(double lon_a, double lat_a, double lon_b, double lat_b) { - double tile_x_a; - double tile_y_a; - double tile_x_b; - double tile_y_b; - double temp; - double x1; - double y1; - double x2; - double y2; - double hyp_len; - double x_len; - double y_len; - double x_step; - double y_step; - double step; - double next_step; - int x; - int y; - int norm_x; - - coords_to_tile(&tile_x_a, &tile_y_a, lon_a, lat_a); - coords_to_tile(&tile_x_b, &tile_y_b, lon_b, lat_b); - - if (tile_x_a > tile_x_b) { - /* We always want the line to go from left to right - swap the ends if it doesn't */ - temp = tile_x_b; - tile_x_b = tile_x_a; - tile_x_a = temp; - temp = tile_y_b; - tile_y_b = tile_y_a; - tile_y_a = temp; - } - - x_len = tile_x_b - tile_x_a; - if (x_len > map_width / 2) { - /* If the line is wider than half the map, assume it - crosses the international date line. - These coordinates get normalised again later */ - tile_x_a += map_width; - temp = tile_x_b; - tile_x_b = tile_x_a; - tile_x_a = temp; - temp = tile_y_b; - tile_y_b = tile_y_a; - tile_y_a = temp; - } - y_len = tile_y_b - tile_y_a; - hyp_len = sqrt(pow(x_len, 2) + pow(y_len, 2)); /* Pythagoras */ - x_step = x_len / hyp_len; - y_step = y_len / hyp_len; - - for (step = 0; step <= hyp_len; step+= 0.4) { - /* Interpolate points 1 tile width apart */ - next_step = step + 0.4; - if (next_step > hyp_len) next_step = hyp_len; - x1 = tile_x_a + ((double)step * x_step); - y1 = tile_y_a + ((double)step * y_step); - x2 = tile_x_a + ((double)next_step * x_step); - y2 = tile_y_a + ((double)next_step * y_step); - - /* The line (x1,y1),(x2,y2) is up to 1 tile width long - x1 will always be <= x2 - We could be smart and figure out the exact tiles intersected, - but for simplicity, treat the coordinates as a bounding box - and expire everything within that box. */ - if (y1 > y2) { - temp = y2; - y2 = y1; - y1 = temp; - } - for (x = x1 - TILE_EXPIRY_LEEWAY; x <= x2 + TILE_EXPIRY_LEEWAY; x ++) { - norm_x = normalise_tile_x_coord(x); - for (y = y1 - TILE_EXPIRY_LEEWAY; y <= y2 + TILE_EXPIRY_LEEWAY; y ++) { - expire_tile(norm_x, y); - } - } - } -} - -/* - * Expire tiles within a bounding box - */ -int expire_tiles_from_bbox(double min_lon, double min_lat, double max_lon, double max_lat) { - double width; - double height; - int min_tile_x; - int min_tile_y; - int max_tile_x; - int max_tile_y; - int iterator_x; - int iterator_y; - int norm_x; - int ret; - double tmp_x; - double tmp_y; - - if (Options->expire_tiles_zoom < 0) return 0; - - width = max_lon - min_lon; - height = max_lat - min_lat; - if (width > HALF_EARTH_CIRCUMFERENCE + 1) { - /* Over half the planet's width within the bounding box - assume the - box crosses the international date line and split it into two boxes */ - ret = expire_tiles_from_bbox(-HALF_EARTH_CIRCUMFERENCE, min_lat, min_lon, max_lat); - ret += expire_tiles_from_bbox(max_lon, min_lat, HALF_EARTH_CIRCUMFERENCE, max_lat); - return ret; - } - - if (width > EXPIRE_TILES_MAX_BBOX) return -1; - if (height > EXPIRE_TILES_MAX_BBOX) return -1; - - - /* Convert the box's Mercator coordinates into tile coordinates */ - coords_to_tile(&tmp_x, &tmp_y, min_lon, max_lat); - min_tile_x = tmp_x - TILE_EXPIRY_LEEWAY; - min_tile_y = tmp_y - TILE_EXPIRY_LEEWAY; - coords_to_tile(&tmp_x, &tmp_y, max_lon, min_lat); - max_tile_x = tmp_x + TILE_EXPIRY_LEEWAY; - max_tile_y = tmp_y + TILE_EXPIRY_LEEWAY; - if (min_tile_x < 0) min_tile_x = 0; - if (min_tile_y < 0) min_tile_y = 0; - if (max_tile_x > map_width) max_tile_x = map_width; - if (max_tile_y > map_width) max_tile_y = map_width; - for (iterator_x = min_tile_x; iterator_x <= max_tile_x; iterator_x ++) { - norm_x = normalise_tile_x_coord(iterator_x); - for (iterator_y = min_tile_y; iterator_y <= max_tile_y; iterator_y ++) { - expire_tile(norm_x, iterator_y); - } - } - return 0; -} - -void expire_tiles_from_nodes_line(struct osmNode * nodes, int count) { - int i; - double last_lat; - double last_lon; - - if (Options->expire_tiles_zoom < 0) return; - if (count < 1) return; - last_lat = nodes[0].lat; - last_lon = nodes[0].lon; - if (count < 2) { - expire_tiles_from_bbox(last_lon, last_lat, last_lon, last_lat); - return; - } - for (i = 1; i < count; i ++) { - expire_tiles_from_line(last_lon, last_lat, nodes[i].lon, nodes[i].lat); - last_lat = nodes[i].lat; - last_lon = nodes[i].lon; - } -} - -/* - * Calculate a bounding box from a list of nodes and expire all tiles within it - */ -void expire_tiles_from_nodes_poly(struct osmNode * nodes, int count, osmid_t osm_id) { - int i; - int got_coords = 0; - double min_lon = 0.0; - double min_lat = 0.0; - double max_lon = 0.0; - double max_lat = 0.0; - - if (Options->expire_tiles_zoom < 0) return; - for (i = 0; i < count; i++) { - if ((! got_coords) || (nodes[i].lon < min_lon)) min_lon = nodes[i].lon; - if ((! got_coords) || (nodes[i].lat < min_lat)) min_lat = nodes[i].lat; - if ((! got_coords) || (nodes[i].lon > max_lon)) max_lon = nodes[i].lon; - if ((! got_coords) || (nodes[i].lat > max_lat)) max_lat = nodes[i].lat; - got_coords = 1; - } - if (got_coords) { - if (expire_tiles_from_bbox(min_lon, min_lat, max_lon, max_lat)) { - /* Bounding box too big - just expire tiles on the line */ - fprintf(stderr, "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID ") - only expiring perimeter\n", max_lon - min_lon, max_lat - min_lat, osm_id); - expire_tiles_from_nodes_line(nodes, count); - } - } -} - -static void expire_tiles_from_xnodes_poly(struct osmNode ** xnodes, int * xcount, osmid_t osm_id) { - int i; - - for (i = 0; xnodes[i]; i++) expire_tiles_from_nodes_poly(xnodes[i], xcount[i], osm_id); -} - -static void expire_tiles_from_xnodes_line(struct osmNode ** xnodes, int * xcount) { - int i; - - for (i = 0; xnodes[i]; i++) expire_tiles_from_nodes_line(xnodes[i], xcount[i]); -} - -void expire_tiles_from_wkt(const char * wkt, osmid_t osm_id) { - struct osmNode ** xnodes; - int * xcount; - int polygon; - int i; - - if (Options->expire_tiles_zoom < 0) return; - if (! parse_wkt(wkt, &xnodes, &xcount, &polygon)) { - if (polygon) expire_tiles_from_xnodes_poly(xnodes, xcount, osm_id); - else expire_tiles_from_xnodes_line(xnodes, xcount); - for (i = 0; xnodes[i]; i++) free(xnodes[i]); - free(xnodes); - free(xcount); - } -} - -/* - * Expire tiles based on an osm element. - * What type of element (node, line, polygon) osm_id refers to depends on - * sql_conn. Each type of table has its own sql_conn and the prepared statement - * get_wkt refers to the appropriate table. - * - * The function returns -1 if expiry is not enabled. Otherwise it returns the number - * of elements that refer to the osm_id. - - */ -int expire_tiles_from_db(PGconn * sql_conn, osmid_t osm_id) { - PGresult * res; - char * wkt; - int i, noElements = 0; - char const *paramValues[1]; - char tmp[16]; - - if (Options->expire_tiles_zoom < 0) return -1; - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, osm_id); - paramValues[0] = tmp; - - /* The prepared statement get_wkt will behave differently depending on the sql_conn - * each table has its own sql_connection with the get_way refering to the approriate table - */ - res = pgsql_execPrepared(sql_conn, "get_wkt", 1, (const char * const *)paramValues, PGRES_TUPLES_OK); - noElements = PQntuples(res); - - for (i = 0; i < noElements; i++) { - wkt = PQgetvalue(res, i, 0); - expire_tiles_from_wkt(wkt, osm_id); - } - PQclear(res); - return noElements; -} - - diff -Nru osm2pgsql-0.82.0/expire-tiles.cpp osm2pgsql-0.88.1/expire-tiles.cpp --- osm2pgsql-0.82.0/expire-tiles.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/expire-tiles.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,527 @@ +/* + * Dirty tile list generation + * + * Steve Hill + * + * Please refer to the OpenPisteMap expire_tiles.py script for a demonstration + * of how to make use of the output: + * https://subversion.nexusuk.org/trac/browser/openpistemap/trunk/scripts/expire_tiles.py + */ + +#include +#include +#include +#include +#include + +#include "expire-tiles.hpp" +#include "options.hpp" +#include "geometry-builder.hpp" +#include "reprojection.hpp" +#include "table.hpp" + +#define EARTH_CIRCUMFERENCE 40075016.68 +#define HALF_EARTH_CIRCUMFERENCE (EARTH_CIRCUMFERENCE / 2) +#define TILE_EXPIRY_LEEWAY 0.1 /* How many tiles worth of space to leave either side of a changed feature */ +#define EXPIRE_TILES_MAX_BBOX 20000 /* Maximum width or height of a bounding box (metres) */ + +namespace { +/* + * We store the dirty tiles in an in-memory tree during runtime + * and dump them out to a file at the end. This allows us to easilly drop + * duplicate tiles from the output. + * + * This data structure consists of a node, representing a tile at zoom level 0, + * which contains 4 pointers to nodes representing each of the child tiles at + * zoom level 1, and so on down the the zoom level specified in + * Options->expire_tiles_zoom. + * + * The memory allowed to this structure is not capped, but daily deltas + * generally produce a few hundred thousand expired tiles at zoom level 17, + * which are easilly accommodated. + */ + +int calc_complete(struct expire_tiles::tile * tile) { + int c; + + c = tile->complete[0][0]; + c += tile->complete[0][1]; + c += tile->complete[1][0]; + c += tile->complete[1][1]; + return c; +} + +void destroy_tree(struct expire_tiles::tile * tree) { + if (! tree) return; + if (tree->subtiles[0][0]) destroy_tree(tree->subtiles[0][0]); + if (tree->subtiles[0][1]) destroy_tree(tree->subtiles[0][1]); + if (tree->subtiles[1][0]) destroy_tree(tree->subtiles[1][0]); + if (tree->subtiles[1][1]) destroy_tree(tree->subtiles[1][1]); + free(tree); +} + +/* + * Mark a tile as dirty. + * Returns the number of subtiles which have all their children marked as dirty. + */ +int _mark_tile(struct expire_tiles::tile ** tree, int x, int y, int zoom, int this_zoom) { + int zoom_diff = zoom - this_zoom; + int rel_x; + int rel_y; + int complete; + + if (! *tree) *tree = (struct expire_tiles::tile *)calloc(1, sizeof(**tree)); + zoom_diff = (zoom - this_zoom) - 1; + rel_x = (x >> zoom_diff) & 1; + rel_y = (y >> zoom_diff) & 1; + if (! (*tree)->complete[rel_x][rel_y]) { + if (zoom_diff <= 0) { + (*tree)->complete[rel_x][rel_y] = 1; + } else { + complete = _mark_tile(&((*tree)->subtiles[rel_x][rel_y]), x, y, zoom, this_zoom + 1); + if (complete >= 4) { + (*tree)->complete[rel_x][rel_y] = 1; + /* We can destroy the subtree to save memory now all the children are dirty */ + destroy_tree((*tree)->subtiles[rel_x][rel_y]); + (*tree)->subtiles[rel_x][rel_y] = NULL; + } + } + } + return calc_complete(*tree); +} + +/* + * Mark a tile as dirty. + * Returns the number of subtiles which have all their children marked as dirty. + */ +int mark_tile(struct expire_tiles::tile ** tree_head, int x, int y, int zoom) { + return _mark_tile(tree_head, x, y, zoom, 0); +} + +void output_dirty_tile_impl(FILE * outfile, int x, int y, int zoom, int min_zoom, int &outcount) { + int y_min; + int x_iter; + int y_iter; + int x_max; + int y_max; + int out_zoom; + int zoom_diff; + + if (zoom > min_zoom) out_zoom = zoom; + else out_zoom = min_zoom; + zoom_diff = out_zoom - zoom; + y_min = y << zoom_diff; + x_max = (x + 1) << zoom_diff; + y_max = (y + 1) << zoom_diff; + for (x_iter = x << zoom_diff; x_iter < x_max; x_iter++) { + for (y_iter = y_min; y_iter < y_max; y_iter++) { + outcount++; + if ((outcount <= 1) || ((outcount % 1000) == 0)) { + fprintf(stderr, "\rWriting dirty tile list (%iK)", outcount / 1000); + fflush(stderr); + } + fprintf(outfile, "%i/%i/%i\n", out_zoom, x_iter, y_iter); + } + } +} + +struct tile_output_file : public expire_tiles::tile_output { + tile_output_file(const std::string &expire_tiles_filename) + : outcount(0) + , outfile(fopen(expire_tiles_filename.c_str(), "a")) { + if (outfile == NULL) { + fprintf(stderr, "Failed to open expired tiles file (%s). Tile expiry list will not be written!\n", strerror(errno)); + } + } + + virtual ~tile_output_file() { + if (outfile) { + fclose(outfile); + } + } + + virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) { + output_dirty_tile_impl(outfile, x, y, zoom, min_zoom, outcount); + } + +private: + int outcount; + FILE *outfile; +}; + +void _output_and_destroy_tree(expire_tiles::tile_output *output, struct expire_tiles::tile * tree, int x, int y, int this_zoom, int min_zoom) { + int sub_x = x << 1; + int sub_y = y << 1; + expire_tiles::tile_output *out; + + if (! tree) return; + + out = output; + if ((tree->complete[0][0]) && output) { + output->output_dirty_tile(sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom); + out = NULL; + } + if (tree->subtiles[0][0]) _output_and_destroy_tree(out, tree->subtiles[0][0], sub_x + 0, sub_y + 0, this_zoom + 1, min_zoom); + + out = output; + if ((tree->complete[0][1]) && output) { + output->output_dirty_tile(sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom); + out = NULL; + } + if (tree->subtiles[0][1]) _output_and_destroy_tree(out, tree->subtiles[0][1], sub_x + 0, sub_y + 1, this_zoom + 1, min_zoom); + + out = output; + if ((tree->complete[1][0]) && output) { + output->output_dirty_tile(sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom); + out = NULL; + } + if (tree->subtiles[1][0]) _output_and_destroy_tree(out, tree->subtiles[1][0], sub_x + 1, sub_y + 0, this_zoom + 1, min_zoom); + + out = output; + if ((tree->complete[1][1]) && output) { + output->output_dirty_tile(sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom); + out = NULL; + } + if (tree->subtiles[1][1]) _output_and_destroy_tree(out, tree->subtiles[1][1], sub_x + 1, sub_y + 1, this_zoom + 1, min_zoom); + + free(tree); +} + +// merge the two trees, destroying b in the process. returns the +// number of completed subtrees. +int _tree_merge(struct expire_tiles::tile **a, + struct expire_tiles::tile **b) { + if (*a == NULL) { + *a = *b; + *b = NULL; + + } else if (*b != NULL) { + for (int x = 0; x < 2; ++x) { + for (int y = 0; y < 2; ++y) { + // if b is complete on a subtree, then the merged tree must + // be complete too. + if ((*b)->complete[x][y]) { + (*a)->complete[x][y] = (*b)->complete[x][y]; + destroy_tree((*a)->subtiles[x][y]); + (*a)->subtiles[x][y] = NULL; + + // but if a is already complete, don't bother moving across + // anything + } else if (!(*a)->complete[x][y]) { + int complete = _tree_merge(&((*a)->subtiles[x][y]), &((*b)->subtiles[x][y])); + + if (complete >= 4) { + (*a)->complete[x][y] = 1; + destroy_tree((*a)->subtiles[x][y]); + (*a)->subtiles[x][y] = NULL; + } + } + + destroy_tree((*b)->subtiles[x][y]); + (*b)->subtiles[x][y] = NULL; + } + } + } + + // count the number complete, so we can return it + int a_complete = 0; + for (int x = 0; x < 2; ++x) { + for (int y = 0; y < 2; ++y) { + if ((*a != NULL) && ((*a)->complete[x][y])) { + ++a_complete; + } + } + } + + return a_complete; +} + +} // anonymous namespace + +void expire_tiles::output_and_destroy(tile_output *output) { + _output_and_destroy_tree(output, dirty, 0, 0, 0, Options->expire_tiles_zoom_min); + dirty = NULL; +} + +void expire_tiles::output_and_destroy() { + if (Options->expire_tiles_zoom >= 0) { + tile_output_file output(Options->expire_tiles_filename); + + output_and_destroy(&output); + } +} + +expire_tiles::~expire_tiles() { + if (dirty != NULL) { + destroy_tree(dirty); + dirty = NULL; + } +} + +expire_tiles::expire_tiles(const struct options_t *options) + : map_width(0), tile_width(0), Options(options), + dirty(NULL) +{ + if (Options->expire_tiles_zoom < 0) return; + map_width = 1 << Options->expire_tiles_zoom; + tile_width = EARTH_CIRCUMFERENCE / map_width; +} + +void expire_tiles::expire_tile(int x, int y) { + mark_tile(&dirty, x, y, Options->expire_tiles_zoom); +} + +int expire_tiles::normalise_tile_x_coord(int x) { + x %= map_width; + if (x < 0) x = (map_width - x) + 1; + return x; +} + +/* + * Expire tiles that a line crosses + */ +void expire_tiles::from_line(double lon_a, double lat_a, double lon_b, double lat_b) { + double tile_x_a; + double tile_y_a; + double tile_x_b; + double tile_y_b; + double temp; + double x1; + double y1; + double x2; + double y2; + double hyp_len; + double x_len; + double y_len; + double x_step; + double y_step; + double step; + double next_step; + int x; + int y; + int norm_x; + + Options->projection->coords_to_tile(&tile_x_a, &tile_y_a, lon_a, lat_a, map_width); + Options->projection->coords_to_tile(&tile_x_b, &tile_y_b, lon_b, lat_b, map_width); + + if (tile_x_a > tile_x_b) { + /* We always want the line to go from left to right - swap the ends if it doesn't */ + temp = tile_x_b; + tile_x_b = tile_x_a; + tile_x_a = temp; + temp = tile_y_b; + tile_y_b = tile_y_a; + tile_y_a = temp; + } + + x_len = tile_x_b - tile_x_a; + if (x_len > map_width / 2) { + /* If the line is wider than half the map, assume it + crosses the international date line. + These coordinates get normalised again later */ + tile_x_a += map_width; + temp = tile_x_b; + tile_x_b = tile_x_a; + tile_x_a = temp; + temp = tile_y_b; + tile_y_b = tile_y_a; + tile_y_a = temp; + } + y_len = tile_y_b - tile_y_a; + hyp_len = sqrt(pow(x_len, 2) + pow(y_len, 2)); /* Pythagoras */ + x_step = x_len / hyp_len; + y_step = y_len / hyp_len; + + for (step = 0; step <= hyp_len; step+= 0.4) { + /* Interpolate points 1 tile width apart */ + next_step = step + 0.4; + if (next_step > hyp_len) next_step = hyp_len; + x1 = tile_x_a + ((double)step * x_step); + y1 = tile_y_a + ((double)step * y_step); + x2 = tile_x_a + ((double)next_step * x_step); + y2 = tile_y_a + ((double)next_step * y_step); + + /* The line (x1,y1),(x2,y2) is up to 1 tile width long + x1 will always be <= x2 + We could be smart and figure out the exact tiles intersected, + but for simplicity, treat the coordinates as a bounding box + and expire everything within that box. */ + if (y1 > y2) { + temp = y2; + y2 = y1; + y1 = temp; + } + for (x = x1 - TILE_EXPIRY_LEEWAY; x <= x2 + TILE_EXPIRY_LEEWAY; x ++) { + norm_x = normalise_tile_x_coord(x); + for (y = y1 - TILE_EXPIRY_LEEWAY; y <= y2 + TILE_EXPIRY_LEEWAY; y ++) { + expire_tile(norm_x, y); + } + } + } +} + +/* + * Expire tiles within a bounding box + */ +int expire_tiles::from_bbox(double min_lon, double min_lat, double max_lon, double max_lat) { + double width; + double height; + int min_tile_x; + int min_tile_y; + int max_tile_x; + int max_tile_y; + int iterator_x; + int iterator_y; + int norm_x; + int ret; + double tmp_x; + double tmp_y; + + if (Options->expire_tiles_zoom < 0) return 0; + + width = max_lon - min_lon; + height = max_lat - min_lat; + if (width > HALF_EARTH_CIRCUMFERENCE + 1) { + /* Over half the planet's width within the bounding box - assume the + box crosses the international date line and split it into two boxes */ + ret = from_bbox(-HALF_EARTH_CIRCUMFERENCE, min_lat, min_lon, max_lat); + ret += from_bbox(max_lon, min_lat, HALF_EARTH_CIRCUMFERENCE, max_lat); + return ret; + } + + if (width > EXPIRE_TILES_MAX_BBOX) return -1; + if (height > EXPIRE_TILES_MAX_BBOX) return -1; + + + /* Convert the box's Mercator coordinates into tile coordinates */ + Options->projection->coords_to_tile(&tmp_x, &tmp_y, min_lon, max_lat, map_width); + min_tile_x = tmp_x - TILE_EXPIRY_LEEWAY; + min_tile_y = tmp_y - TILE_EXPIRY_LEEWAY; + Options->projection->coords_to_tile(&tmp_x, &tmp_y, max_lon, min_lat, map_width); + max_tile_x = tmp_x + TILE_EXPIRY_LEEWAY; + max_tile_y = tmp_y + TILE_EXPIRY_LEEWAY; + if (min_tile_x < 0) min_tile_x = 0; + if (min_tile_y < 0) min_tile_y = 0; + if (max_tile_x > map_width) max_tile_x = map_width; + if (max_tile_y > map_width) max_tile_y = map_width; + for (iterator_x = min_tile_x; iterator_x <= max_tile_x; iterator_x ++) { + norm_x = normalise_tile_x_coord(iterator_x); + for (iterator_y = min_tile_y; iterator_y <= max_tile_y; iterator_y ++) { + expire_tile(norm_x, iterator_y); + } + } + return 0; +} + +void expire_tiles::from_nodes_line(const nodelist_t &nodes) +{ + if (Options->expire_tiles_zoom < 0 || nodes.empty()) + return; + + if (nodes.size() == 1) { + from_bbox(nodes[0].lon, nodes[0].lat, nodes[0].lon, nodes[0].lat); + } else { + for (size_t i = 1; i < nodes.size(); ++i) + from_line(nodes[i-1].lon, nodes[i-1].lat, nodes[i].lon, nodes[i].lat); + } +} + +/* + * Calculate a bounding box from a list of nodes and expire all tiles within it + */ +void expire_tiles::from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id) +{ + if (Options->expire_tiles_zoom < 0 || nodes.empty()) + return; + + double min_lon = nodes[0].lon; + double min_lat = nodes[0].lat; + double max_lon = nodes[0].lon; + double max_lat = nodes[0].lat; + + for (size_t i = 1; i < nodes.size(); ++i) { + if (nodes[i].lon < min_lon) min_lon = nodes[i].lon; + if (nodes[i].lat < min_lat) min_lat = nodes[i].lat; + if (nodes[i].lon > max_lon) max_lon = nodes[i].lon; + if (nodes[i].lat > max_lat) max_lat = nodes[i].lat; + } + + if (from_bbox(min_lon, min_lat, max_lon, max_lat)) { + /* Bounding box too big - just expire tiles on the line */ + fprintf(stderr, "\rLarge polygon (%.0f x %.0f metres, OSM ID %" PRIdOSMID ") - only expiring perimeter\n", max_lon - min_lon, max_lat - min_lat, osm_id); + from_nodes_line(nodes); + } +} + +void expire_tiles::from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id) +{ + for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) + from_nodes_poly(*it, osm_id); +} + +void expire_tiles::from_xnodes_line(const multinodelist_t &xnodes) +{ + for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) + from_nodes_line(*it); +} + +void expire_tiles::from_wkt(const char * wkt, osmid_t osm_id) +{ + if (Options->expire_tiles_zoom < 0) return; + + multinodelist_t xnodes; + int polygon; + + if (!geometry_builder::parse_wkt(wkt, xnodes, &polygon)) { + if (polygon) + from_xnodes_poly(xnodes, osm_id); + else + from_xnodes_line(xnodes); + } +} + +/* + * Expire tiles based on an osm element. + * What type of element (node, line, polygon) osm_id refers to depends on + * sql_conn. Each type of table has its own sql_conn and the prepared statement + * get_wkt refers to the appropriate table. + * + * The function returns -1 if expiry is not enabled. Otherwise it returns the number + * of elements that refer to the osm_id. + + */ +int expire_tiles::from_db(table_t* table, osmid_t osm_id) { + //bail if we dont care about expiry + if (Options->expire_tiles_zoom < 0) + return -1; + + //grab the geom for this id + boost::shared_ptr wkts = table->get_wkt_reader(osm_id); + + //dirty the stuff + const char* wkt = NULL; + while((wkt = wkts->get_next())) + from_wkt(wkt, osm_id); + + //return how many rows were affected + return wkts->get_count(); +} + +void expire_tiles::merge_and_destroy(expire_tiles &other) { + if (map_width != other.map_width) { + throw std::runtime_error((boost::format("Unable to merge tile expiry sets when " + "map_width does not match: %1% != %2%.") + % map_width % other.map_width).str()); + } + + if (tile_width != other.tile_width) { + throw std::runtime_error((boost::format("Unable to merge tile expiry sets when " + "tile_width does not match: %1% != %2%.") + % tile_width % other.tile_width).str()); + } + + _tree_merge(&dirty, &other.dirty); + + destroy_tree(other.dirty); + other.dirty = NULL; +} diff -Nru osm2pgsql-0.82.0/expire-tiles.h osm2pgsql-0.88.1/expire-tiles.h --- osm2pgsql-0.82.0/expire-tiles.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/expire-tiles.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -#ifndef EXPIRE_TILES_H -#define EXPIRE_TILES_H - -#include "output.h" - -void expire_tiles_init(const struct output_options *options); -void expire_tiles_stop(void); -int expire_tiles_from_bbox(double min_lon, double min_lat, double max_lon, double max_lat); -void expire_tiles_from_nodes_line(struct osmNode * nodes, int count); -void expire_tiles_from_nodes_poly(struct osmNode * nodes, int count, osmid_t osm_id); -void expire_tiles_from_wkt(const char * wkt, osmid_t osm_id); -int expire_tiles_from_db(PGconn * sql_conn, osmid_t osm_id); - -#endif diff -Nru osm2pgsql-0.82.0/expire-tiles.hpp osm2pgsql-0.88.1/expire-tiles.hpp --- osm2pgsql-0.82.0/expire-tiles.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/expire-tiles.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,64 @@ +#ifndef EXPIRE_TILES_H +#define EXPIRE_TILES_H + +#include "osmtypes.hpp" + +#include + +class table_t; +struct options_t; + +struct expire_tiles : public boost::noncopyable { + explicit expire_tiles(const options_t *options); + ~expire_tiles(); + + //TODO: copy constructor + + int from_bbox(double min_lon, double min_lat, double max_lon, double max_lat); + void from_nodes_line(const nodelist_t &nodes); + void from_nodes_poly(const nodelist_t &nodes, osmid_t osm_id); + void from_wkt(const char * wkt, osmid_t osm_id); + int from_db(table_t* table, osmid_t osm_id); + + struct tile { + int complete[2][2]; + struct tile* subtiles[2][2]; + }; + + /* customisable tile output. this can be passed into the + * `output_and_destroy` function to override output to a file. + * this is primarily useful for testing. + */ + struct tile_output { + virtual ~tile_output() {} + // dirty a tile at x, y & zoom, and all descendants of that + // tile at the given zoom if zoom < min_zoom. + virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) = 0; + }; + + // output the list of expired tiles to a file. note that this + // consumes the list of expired tiles destructively. + void output_and_destroy(); + + // output the list of expired tiles using a `tile_output` + // functor. this consumes the list of expired tiles destructively. + void output_and_destroy(tile_output *output); + + // merge the list of expired tiles in the other object into this + // object, destroying the list in the other object. + void merge_and_destroy(expire_tiles &); + +private: + void expire_tile(int x, int y); + int normalise_tile_x_coord(int x); + void from_line(double lon_a, double lat_a, double lon_b, double lat_b); + void from_xnodes_poly(const multinodelist_t &xnodes, osmid_t osm_id); + void from_xnodes_line(const multinodelist_t &xnodes); + + int map_width; + double tile_width; + const options_t *Options; + struct tile *dirty; +}; + +#endif diff -Nru osm2pgsql-0.82.0/geometry-builder.cpp osm2pgsql-0.88.1/geometry-builder.cpp --- osm2pgsql-0.82.0/geometry-builder.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/geometry-builder.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,786 @@ +/* +#----------------------------------------------------------------------------- +# Part of osm2pgsql utility +#----------------------------------------------------------------------------- +# By Artem Pavlenko, Copyright 2007 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__CYGWIN__) +#define GEOS_INLINE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace geos::geom; +using namespace geos::io; +using namespace geos::util; +using namespace geos::operation::linemerge; + +#include "geometry-builder.hpp" + +typedef std::auto_ptr geom_ptr; + +namespace { + +void coords2nodes(CoordinateSequence * coords, nodelist_t &nodes) +{ + size_t num_coords = coords->getSize(); + nodes.reserve(num_coords); + + for (size_t i = 0; i < num_coords; i++) { + Coordinate coord = coords->getAt(i); + nodes.push_back(osmNode(coord.x, coord.y)); + } +} + +struct polygondata +{ + Polygon* polygon; + LinearRing* ring; + double area; + int iscontained; + unsigned containedbyid; +}; + +struct polygondata_comparearea { + bool operator()(const polygondata& lhs, const polygondata& rhs) { + return lhs.area > rhs.area; + } +}; + +} // anonymous namespace + +geometry_builder::maybe_wkt_t geometry_builder::get_wkt_simple(const nodelist_t &nodes, int polygon) const +{ + GeometryFactory gf; + std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + + try + { + for (nodelist_t::const_iterator it = nodes.begin(); it != nodes.end(); ++it) { + coords->add(Coordinate(it->lon, it->lat), 0); + } + + maybe_wkt_t wkt(new geometry_builder::wkt_t()); + geom_ptr geom; + if (polygon && (coords->getSize() >= 4) && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0)))) { + std::auto_ptr shell(gf.createLinearRing(coords.release())); + geom = geom_ptr(gf.createPolygon(shell.release(), new std::vector)); + if (!geom->isValid()) { + if (excludepoly) { + throw std::runtime_error("Excluding broken polygon."); + } else { + geom = geom_ptr(geom->buffer(0)); + } + } + geom->normalize(); // Fix direction of ring + wkt->area = geom->getArea(); + } else { + if (coords->getSize() < 2) + throw std::runtime_error("Excluding degenerate line."); + geom = geom_ptr(gf.createLineString(coords.release())); + wkt->area = 0; + } + + wkt->geom = WKTWriter().write(geom.get()); + return wkt; + } + catch (const std::bad_alloc&) + { + std::cerr << std::endl << "Exception caught processing way. You are likelly running out of memory." << std::endl; + std::cerr << "Try in slim mode, using -s parameter." << std::endl; + } + catch (const std::runtime_error& e) + { + //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl; + } + catch (...) + { + std::cerr << std::endl << "Exception caught processing way" << std::endl; + } + + return maybe_wkt_t(); +} + +geometry_builder::maybe_wkts_t geometry_builder::get_wkt_split(const nodelist_t &nodes, int polygon, double split_at) const +{ + GeometryFactory gf; + std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + WKTWriter writer; + //TODO: use count to get some kind of hint of how much we should reserve? + maybe_wkts_t wkts(new std::vector); + + try + { + for (nodelist_t::const_iterator it = nodes.begin(); it != nodes.end(); ++it) { + coords->add(Coordinate(it->lon, it->lat), 0); + } + + geom_ptr geom; + if (polygon && (coords->getSize() >= 4) && (coords->getAt(coords->getSize() - 1).equals2D(coords->getAt(0)))) { + std::auto_ptr shell(gf.createLinearRing(coords.release())); + geom = geom_ptr(gf.createPolygon(shell.release(), new std::vector)); + if (!geom->isValid()) { + if (excludepoly) { + throw std::runtime_error("Excluding broken polygon."); + } else { + geom = geom_ptr(geom->buffer(0)); + } + } + geom->normalize(); // Fix direction of ring + + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(geom.get()); + wkts->back().area = geom->getArea(); + + } else { + if (coords->getSize() < 2) + throw std::runtime_error("Excluding degenerate line."); + + double distance = 0; + std::auto_ptr segment; + segment = std::auto_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + segment->add(coords->getAt(0)); + for(unsigned i=1; igetSize(); i++) { + const Coordinate this_pt = coords->getAt(i); + const Coordinate prev_pt = coords->getAt(i-1); + const double delta = this_pt.distance(prev_pt); + assert(!std::isnan(delta)); + // figure out if the addition of this point would take the total + // length of the line in `segment` over the `split_at` distance. + + if (distance + delta > split_at) { + const size_t splits = std::floor((distance + delta) / split_at); + // use the splitting distance to split the current segment up + // into as many parts as necessary to keep each part below + // the `split_at` distance. + for (size_t i = 0; i < splits; ++i) { + double frac = (double(i + 1) * split_at - distance) / delta; + const Coordinate interpolated(frac * (this_pt.x - prev_pt.x) + prev_pt.x, + frac * (this_pt.y - prev_pt.y) + prev_pt.y); + segment->add(interpolated); + geom_ptr geom = geom_ptr(gf.createLineString(segment.release())); + + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(geom.get()); + wkts->back().area = 0; + + segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + segment->add(interpolated); + } + // reset the distance based on the final splitting point for + // the next iteration. + distance = segment->getAt(0).distance(this_pt); + + } else { + // if not split then just push this point onto the sequence + // being saved up. + distance += delta; + } + + // always add this point + segment->add(this_pt); + + // on the last iteration, close out the line. + if (i == coords->getSize()-1) { + geom_ptr geom = geom_ptr(gf.createLineString(segment.release())); + + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(geom.get()); + wkts->back().area = 0; + + segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + } + } + } + } + catch (const std::bad_alloc&) + { + std::cerr << std::endl << "Exception caught processing way. You are likely running out of memory." << std::endl; + std::cerr << "Try in slim mode, using -s parameter." << std::endl; + } + catch (const std::runtime_error& e) + { + //std::cerr << std::endl << "Exception caught processing way: " << e.what() << std::endl; + } + catch (...) + { + std::cerr << std::endl << "Exception caught processing way" << std::endl; + } + return wkts; +} + +int geometry_builder::parse_wkt(const char * wkt, multinodelist_t &nodes, int *polygon) { + GeometryFactory gf; + WKTReader reader(&gf); + std::string wkt_string(wkt); + GeometryCollection * gc; + CoordinateSequence * coords; + size_t num_geometries; + + *polygon = 0; + try { + Geometry * geometry = reader.read(wkt_string); + switch (geometry->getGeometryTypeId()) { + // Single geometries + case GEOS_POLYGON: + // Drop through + case GEOS_LINEARRING: + *polygon = 1; + // Drop through + case GEOS_POINT: + // Drop through + case GEOS_LINESTRING: + nodes.push_back(nodelist_t()); + coords = geometry->getCoordinates(); + coords2nodes(coords, nodes.back()); + delete coords; + break; + // Geometry collections + case GEOS_MULTIPOLYGON: + *polygon = 1; + // Drop through + case GEOS_MULTIPOINT: + // Drop through + case GEOS_MULTILINESTRING: + gc = dynamic_cast(geometry); + num_geometries = gc->getNumGeometries(); + nodes.assign(num_geometries, nodelist_t()); + for (size_t i = 0; i < num_geometries; i++) { + const Geometry *subgeometry = gc->getGeometryN(i); + coords = subgeometry->getCoordinates(); + coords2nodes(coords, nodes[i]); + delete coords; + } + break; + default: + std::cerr << std::endl << "unexpected object type while processing PostGIS data" << std::endl; + delete geometry; + return -1; + } + delete geometry; + } catch (...) { + std::cerr << std::endl << "Exception caught parsing PostGIS data" << std::endl; + return -1; + } + return 0; +} + +geometry_builder::maybe_wkts_t geometry_builder::build_polygons(const multinodelist_t &xnodes, + bool enable_multi, osmid_t osm_id) const +{ + std::auto_ptr > lines(new std::vector); + GeometryFactory gf; + geom_ptr geom; + geos::geom::prep::PreparedGeometryFactory pgf; + + maybe_wkts_t wkts(new std::vector); + + try + { + for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) { + std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + for (nodelist_t::const_iterator node = it->begin(); node != it->end(); ++node) { + Coordinate c; + c.x = node->lon; + c.y = node->lat; + coords->add(c, 0); + } + if (coords->getSize() > 1) { + geom = geom_ptr(gf.createLineString(coords.release())); + lines->push_back(geom.release()); + } + } + + //geom_ptr segment(0); + geom_ptr mline (gf.createMultiLineString(lines.release())); + //geom_ptr noded (segment->Union(mline.get())); + LineMerger merger; + //merger.add(noded.get()); + merger.add(mline.get()); + std::auto_ptr > merged(merger.getMergedLineStrings()); + WKTWriter writer; + + // Procces ways into lines or simple polygon list + std::vector polys(merged->size()); + + unsigned totalpolys = 0; + for (unsigned i=0 ;i < merged->size(); ++i) + { + std::auto_ptr pline ((*merged ) [i]); + if (pline->getNumPoints() > 3 && pline->isClosed()) + { + polys[totalpolys].polygon = gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0); + polys[totalpolys].ring = gf.createLinearRing(pline->getCoordinates()); + polys[totalpolys].area = polys[totalpolys].polygon->getArea(); + polys[totalpolys].iscontained = 0; + polys[totalpolys].containedbyid = 0; + if (polys[totalpolys].area > 0.0) + totalpolys++; + else { + delete(polys[totalpolys].polygon); + delete(polys[totalpolys].ring); + } + } + } + + if (totalpolys) + { + std::sort(polys.begin(), polys.begin() + totalpolys, polygondata_comparearea()); + + unsigned toplevelpolygons = 0; + int istoplevelafterall; + + for (unsigned i=0 ;i < totalpolys; ++i) + { + if (polys[i].iscontained != 0) continue; + toplevelpolygons++; + const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon); + + for (unsigned j=i+1; j < totalpolys; ++j) + { + // Does preparedtoplevelpolygon contain the smaller polygon[j]? + if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon)) + { + // are we in a [i] contains [k] contains [j] situation + // which would actually make j top level + istoplevelafterall = 0; + for (unsigned k=i+1; k < j; ++k) + { + if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon)) + { + istoplevelafterall = 1; + break; + } +#if 0 + else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon)) + { + // FIXME: This code does not work as intended + // It should be setting the polys[k].ring in order to update this object + // but the value of polys[k].polygon calculated is normally NULL + + // Add polygon this polygon (j) to k since they intersect + // Mark ourselfs to be dropped (2), delete the original k + Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon); + delete(polys[k].polygon); + polys[k].polygon = dynamic_cast(polyunion); + polys[j].iscontained = 2; // Drop + istoplevelafterall = 2; + break; + } +#endif + } + if (istoplevelafterall == 0) + { + polys[j].iscontained = 1; + polys[j].containedbyid = i; + } + } + } + pgf.destroy(preparedtoplevelpolygon); + } + // polys now is a list of polygons tagged with which ones are inside each other + + // List of polygons for multipolygon + std::auto_ptr > polygons(new std::vector); + + // For each top level polygon create a new polygon including any holes + for (unsigned i=0 ;i < totalpolys; ++i) + { + if (polys[i].iscontained != 0) continue; + + // List of holes for this top level polygon + std::auto_ptr > interior(new std::vector); + for (unsigned j=i+1; j < totalpolys; ++j) + { + if (polys[j].iscontained == 1 && polys[j].containedbyid == i) + { + interior->push_back(polys[j].ring); + } + } + + Polygon* poly(gf.createPolygon(polys[i].ring, interior.release())); + poly->normalize(); + polygons->push_back(poly); + } + + // Make a multipolygon if required + if ((toplevelpolygons > 1) && enable_multi) + { + geom_ptr multipoly(gf.createMultiPolygon(polygons.release())); + if (!multipoly->isValid() && (excludepoly == 0)) { + multipoly = geom_ptr(multipoly->buffer(0)); + } + multipoly->normalize(); + + if ((excludepoly == 0) || (multipoly->isValid())) + { + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(multipoly.get()); + wkts->back().area = multipoly->getArea(); + } + } + else + { + for(unsigned i=0; i(polygons->at(i)); + if (!poly->isValid() && (excludepoly == 0)) { + poly = dynamic_cast(poly->buffer(0)); + poly->normalize(); + } + if ((excludepoly == 0) || (poly->isValid())) + { + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(poly); + wkts->back().area = poly->getArea(); + } + delete(poly); + } + } + } + + for (unsigned i=0; i < totalpolys; ++i) + { + delete(polys[i].polygon); + } + }//TODO: don't show in message id when osm_id == -1 + catch (const std::exception& e) + { + std::cerr << std::endl << "Standard exception processing way_id="<< osm_id << ": " << e.what() << std::endl; + } + catch (...) + { + std::cerr << std::endl << "Exception caught processing way id=" << osm_id << std::endl; + } + + return wkts; +} + +geometry_builder::maybe_wkt_t geometry_builder::build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const +{ + std::auto_ptr > lines(new std::vector); + GeometryFactory gf; + geom_ptr geom; + + maybe_wkt_t wkt(new geometry_builder::wkt_t()); + + try + { + for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) { + std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + for (nodelist_t::const_iterator node = it->begin(); node != it->end(); ++node) { + Coordinate c; + c.x = node->lon; + c.y = node->lat; + coords->add(c, 0); + } + if (coords->getSize() > 1) { + geom = geom_ptr(gf.createLineString(coords.release())); + lines->push_back(geom.release()); + } + } + + //geom_ptr segment(0); + geom_ptr mline (gf.createMultiLineString(lines.release())); + //geom_ptr noded (segment->Union(mline.get())); + + WKTWriter writer; + wkt->geom = writer.write(mline.get()); + wkt->area = 0; + }//TODO: don't show in message id when osm_id == -1 + catch (const std::exception& e) + { + std::cerr << std::endl << "Standard exception processing way_id="<< osm_id << ": " << e.what() << std::endl; + } + catch (...) + { + std::cerr << std::endl << "Exception caught processing way id=" << osm_id << std::endl; + } + return wkt; +} + +geometry_builder::maybe_wkts_t geometry_builder::build_both(const multinodelist_t &xnodes, + int make_polygon, int enable_multi, + double split_at, osmid_t osm_id) const +{ + std::auto_ptr > lines(new std::vector); + GeometryFactory gf; + geom_ptr geom; + geos::geom::prep::PreparedGeometryFactory pgf; + maybe_wkts_t wkts(new std::vector); + + + try + { + for (multinodelist_t::const_iterator it = xnodes.begin(); it != xnodes.end(); ++it) { + std::auto_ptr coords(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + for (nodelist_t::const_iterator node = it->begin(); node != it->end(); ++node) { + Coordinate c; + c.x = node->lon; + c.y = node->lat; + coords->add(c, 0); + } + if (coords->getSize() > 1) { + geom = geom_ptr(gf.createLineString(coords.release())); + lines->push_back(geom.release()); + } + } + + //geom_ptr segment(0); + geom_ptr mline (gf.createMultiLineString(lines.release())); + //geom_ptr noded (segment->Union(mline.get())); + LineMerger merger; + //merger.add(noded.get()); + merger.add(mline.get()); + std::auto_ptr > merged(merger.getMergedLineStrings()); + WKTWriter writer; + + // Procces ways into lines or simple polygon list + std::vector polys(merged->size()); + + unsigned totalpolys = 0; + for (unsigned i=0 ;i < merged->size(); ++i) + { + std::auto_ptr pline ((*merged ) [i]); + if (make_polygon && pline->getNumPoints() > 3 && pline->isClosed()) + { + polys[totalpolys].polygon = gf.createPolygon(gf.createLinearRing(pline->getCoordinates()),0); + polys[totalpolys].ring = gf.createLinearRing(pline->getCoordinates()); + polys[totalpolys].area = polys[totalpolys].polygon->getArea(); + polys[totalpolys].iscontained = 0; + polys[totalpolys].containedbyid = 0; + if (polys[totalpolys].area > 0.0) + totalpolys++; + else { + delete(polys[totalpolys].polygon); + delete(polys[totalpolys].ring); + } + } + else + { + //std::cerr << "polygon(" << osm_id << ") is no good: points(" << pline->getNumPoints() << "), closed(" << pline->isClosed() << "). " << writer.write(pline.get()) << std::endl; + double distance = 0; + std::auto_ptr segment; + segment = std::auto_ptr(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + segment->add(pline->getCoordinateN(0)); + for(unsigned i=1; igetNumPoints(); i++) { + segment->add(pline->getCoordinateN(i)); + distance += pline->getCoordinateN(i).distance(pline->getCoordinateN(i-1)); + if ((distance >= split_at) || (i == pline->getNumPoints()-1)) { + geom_ptr geom = geom_ptr(gf.createLineString(segment.release())); + + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(geom.get()); + wkts->back().area = 0; + + segment.reset(gf.getCoordinateSequenceFactory()->create((size_t)0, (size_t)2)); + distance=0; + segment->add(pline->getCoordinateN(i)); + } + } + //std::string text = writer.write(pline.get()); + //wkts.push_back(text); + //areas.push_back(0.0); + //wkt_size++; + } + } + + if (totalpolys) + { + std::sort(polys.begin(), polys.begin() + totalpolys, polygondata_comparearea()); + + unsigned toplevelpolygons = 0; + int istoplevelafterall; + + for (unsigned i=0 ;i < totalpolys; ++i) + { + if (polys[i].iscontained != 0) continue; + toplevelpolygons++; + const geos::geom::prep::PreparedGeometry* preparedtoplevelpolygon = pgf.create(polys[i].polygon); + + for (unsigned j=i+1; j < totalpolys; ++j) + { + // Does preparedtoplevelpolygon contain the smaller polygon[j]? + if (polys[j].containedbyid == 0 && preparedtoplevelpolygon->contains(polys[j].polygon)) + { + // are we in a [i] contains [k] contains [j] situation + // which would actually make j top level + istoplevelafterall = 0; + for (unsigned k=i+1; k < j; ++k) + { + if (polys[k].iscontained && polys[k].containedbyid == i && polys[k].polygon->contains(polys[j].polygon)) + { + istoplevelafterall = 1; + break; + } +#if 0 + else if (polys[k].polygon->intersects(polys[j].polygon) || polys[k].polygon->touches(polys[j].polygon)) + { + // FIXME: This code does not work as intended + // It should be setting the polys[k].ring in order to update this object + // but the value of polys[k].polygon calculated is normally NULL + + // Add polygon this polygon (j) to k since they intersect + // Mark ourselfs to be dropped (2), delete the original k + Geometry* polyunion = polys[k].polygon->Union(polys[j].polygon); + delete(polys[k].polygon); + polys[k].polygon = dynamic_cast(polyunion); + polys[j].iscontained = 2; // Drop + istoplevelafterall = 2; + break; + } +#endif + } + if (istoplevelafterall == 0) + { + polys[j].iscontained = 1; + polys[j].containedbyid = i; + } + } + } + pgf.destroy(preparedtoplevelpolygon); + } + // polys now is a list of polygons tagged with which ones are inside each other + + // List of polygons for multipolygon + std::auto_ptr > polygons(new std::vector); + + // For each top level polygon create a new polygon including any holes + for (unsigned i=0 ;i < totalpolys; ++i) + { + if (polys[i].iscontained != 0) continue; + + // List of holes for this top level polygon + std::auto_ptr > interior(new std::vector); + for (unsigned j=i+1; j < totalpolys; ++j) + { + if (polys[j].iscontained == 1 && polys[j].containedbyid == i) + { + interior->push_back(polys[j].ring); + } + } + + Polygon* poly(gf.createPolygon(polys[i].ring, interior.release())); + poly->normalize(); + polygons->push_back(poly); + } + + // Make a multipolygon if required + if ((toplevelpolygons > 1) && enable_multi) + { + geom_ptr multipoly(gf.createMultiPolygon(polygons.release())); + if (!multipoly->isValid() && (excludepoly == 0)) { + multipoly = geom_ptr(multipoly->buffer(0)); + } + multipoly->normalize(); + + if ((excludepoly == 0) || (multipoly->isValid())) + { + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(multipoly.get()); + wkts->back().area = multipoly->getArea(); + } + } + else + { + for(unsigned i=0; i(polygons->at(i)); + if (!poly->isValid() && (excludepoly == 0)) { + poly = dynamic_cast(poly->buffer(0)); + poly->normalize(); + } + if ((excludepoly == 0) || (poly->isValid())) + { + //copy of an empty one should be cheapest + wkts->push_back(geometry_builder::wkt_t()); + //then we set on the one we already have + wkts->back().geom = writer.write(poly); + wkts->back().area = poly->getArea(); + } + delete(poly); + } + } + } + + for (unsigned i=0; i < totalpolys; ++i) + { + delete(polys[i].polygon); + } + }//TODO: don't show in message id when osm_id == -1 + catch (const std::exception& e) + { + std::cerr << std::endl << "Standard exception processing relation id="<< osm_id << ": " << e.what() << std::endl; + } + catch (...) + { + std::cerr << std::endl << "Exception caught processing relation id=" << osm_id << std::endl; + } + + return wkts; +} + +void geometry_builder::set_exclude_broken_polygon(int exclude) +{ + excludepoly = exclude; +} + +geometry_builder::geometry_builder() + : excludepoly(0) { +} + +geometry_builder::~geometry_builder() { +} diff -Nru osm2pgsql-0.82.0/geometry-builder.hpp osm2pgsql-0.88.1/geometry-builder.hpp --- osm2pgsql-0.82.0/geometry-builder.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/geometry-builder.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,66 @@ +/* +#----------------------------------------------------------------------------- +# Part of osm2pgsql utility +#----------------------------------------------------------------------------- +# By Artem Pavlenko, Copyright 2007 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#ifndef GEOMETRY_BUILDER_H +#define GEOMETRY_BUILDER_H + +#include "osmtypes.hpp" + +#include +#include +#include +#include + +struct geometry_builder : public boost::noncopyable +{ + struct wkt_t + { + wkt_t(const std::string& geom, const double& area):geom(geom),area(area){} + wkt_t():geom(""),area(0){} + std::string geom; + double area; + }; + + // type to represent an optional return of WKT-encoded geometry + typedef boost::shared_ptr maybe_wkt_t; + typedef boost::shared_ptr > maybe_wkts_t; + typedef std::vector::const_iterator wkt_itr; + + geometry_builder(); + ~geometry_builder(); + + static int parse_wkt(const char *wkt, multinodelist_t &nodes, int *polygon); + maybe_wkt_t get_wkt_simple(const nodelist_t &nodes, int polygon) const; + maybe_wkts_t get_wkt_split(const nodelist_t &nodes, int polygon, double split_at) const; + maybe_wkts_t build_both(const multinodelist_t &xnodes, int make_polygon, + int enable_multi, double split_at, osmid_t osm_id = -1) const; + maybe_wkts_t build_polygons(const multinodelist_t &xnodes, bool enable_multi, osmid_t osm_id = -1) const; + // Used by gazetteer. Outputting a multiline, it only ever returns one WKT + maybe_wkt_t build_multilines(const multinodelist_t &xnodes, osmid_t osm_id) const; + + void set_exclude_broken_polygon(int exclude); + +private: + int excludepoly; +}; + +#endif diff -Nru osm2pgsql-0.82.0/geometry-processor.cpp osm2pgsql-0.88.1/geometry-processor.cpp --- osm2pgsql-0.82.0/geometry-processor.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/geometry-processor.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,139 @@ +#include "geometry-processor.hpp" +#include "processor-line.hpp" +#include "processor-point.hpp" +#include "processor-polygon.hpp" +#include "middle.hpp" +#include "options.hpp" +#include "reprojection.hpp" + +#include +#include +#include +#include + +boost::shared_ptr geometry_processor::create(const std::string &type, + const options_t *options) { + boost::shared_ptr ptr; + int srid = options->projection->project_getprojinfo()->srs; + + if (type == "point") { + ptr = boost::make_shared(srid); + } + else if (type == "line") { + ptr = boost::make_shared(srid); + } + else if (type == "polygon") { + ptr = boost::make_shared(srid, options->enable_multi); + } + else { + throw std::runtime_error((boost::format("Unable to construct geometry processor " + "because type `%1%' is not known.") + % type).str()); + } + + return ptr; +} + +geometry_processor::geometry_processor(int srid, const std::string &type, unsigned int interests) + : m_srid(srid), m_type(type), m_interests(interests) { +} + +geometry_processor::~geometry_processor() { +} + +int geometry_processor::srid() const { + return m_srid; +} + +const std::string &geometry_processor::column_type() const { + return m_type; +} + +unsigned int geometry_processor::interests() const { + return m_interests; +} + +bool geometry_processor::interests(unsigned int interested) const { + return (interested & m_interests) == interested; +} + +geometry_builder::maybe_wkt_t geometry_processor::process_node(double lat, double lon) { + return geometry_builder::maybe_wkt_t(); +} + +geometry_builder::maybe_wkt_t geometry_processor::process_way(const nodelist_t &nodes) { + return geometry_builder::maybe_wkt_t(); +} + +geometry_builder::maybe_wkts_t geometry_processor::process_relation(const multinodelist_t &nodes) { + return geometry_builder::maybe_wkts_t(); +} + +way_helper::way_helper() +{ +} +way_helper::~way_helper() +{ +} +size_t way_helper::set(const idlist_t &node_ids, const middle_query_t *mid) +{ + node_cache.clear(); + mid->nodes_get_list(node_cache, node_ids); + + // equivalent to returning node_count for complete ways, different for partial extracts + return node_cache.size(); +} + +relation_helper::relation_helper() +{ +} + +relation_helper::~relation_helper() +{ +} + +size_t relation_helper::set(const memberlist_t *member_list, const middle_t* mid) +{ + // cleanup + input_way_ids.clear(); + ways.clear(); + tags.clear(); + nodes.clear(); + roles.clear(); + + //keep a few things + members = member_list; + + //grab the way members' ids + input_way_ids.reserve(member_list->size()); + for (memberlist_t::const_iterator it = members->begin(); it != members->end(); ++it) { + if(it->type == OSMTYPE_WAY) + input_way_ids.push_back(it->id); + } + + //if we didn't end up using any we'll bail + if (input_way_ids.empty()) + return 0; + + //get the nodes of the ways + mid->ways_get_list(input_way_ids, ways, tags, nodes); + + //grab the roles of each way + roles.reserve(ways.size()); + size_t memberpos = 0; + for (idlist_t::const_iterator it = ways.begin(); it != ways.end(); ++it) { + while (memberpos < members->size()) { + if (members->at(memberpos).id == *it) { + roles.push_back(&(members->at(memberpos).role)); + memberpos++; + break; + } + memberpos++; + } + } + + //mark the ends of each so whoever uses them will know where they end.. + superseeded.resize(ways.size()); + + return ways.size(); +} diff -Nru osm2pgsql-0.82.0/geometry-processor.hpp osm2pgsql-0.88.1/geometry-processor.hpp --- osm2pgsql-0.82.0/geometry-processor.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/geometry-processor.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,101 @@ +#ifndef GEOMETRY_PROCESSOR_HPP +#define GEOMETRY_PROCESSOR_HPP + +#include +#include +#include +#include +#include "geometry-builder.hpp" +#include "osmtypes.hpp" + +struct middle_query_t; +struct middle_t; +struct options_t; + +struct geometry_processor { + // factory method for creating various types of geometry processors either by name or by geometry column type + static boost::shared_ptr create(const std::string &type, + const options_t *options); + + virtual ~geometry_processor(); + + enum interest { + interest_NONE = 0, + interest_node = 1, + interest_way = 2, + interest_relation = 4, + interest_ALL = 7 + }; + + // return bit-mask of the type of elements this processor is + // interested in. + unsigned int interests() const; + + // return true if provided intrest is an interest of this processor + bool interests(unsigned int interested) const; + + // the postgis column type for the kind of geometry (i.e: POINT, + // LINESTRING, etc...) that this processor outputs + const std::string &column_type() const; + + // process a node, optionally returning a WKT string describing + // geometry to be inserted into the table. + virtual geometry_builder::maybe_wkt_t process_node(double lat, double lon); + + // process a way + // position data and optionally returning WKT-encoded geometry + // for insertion into the table. + virtual geometry_builder::maybe_wkt_t process_way(const nodelist_t &nodes); + + // process a way, taking a middle query object to get way and + // node position data. optionally returns a WKT-encoded geometry + // for insertion into the table. + virtual geometry_builder::maybe_wkts_t process_relation(const multinodelist_t &nodes); + + // returns the SRID of the output geometry. + int srid() const; + +protected: + // SRID of the geometry output + const int m_srid; + + // WKT type of the geometry output + const std::string m_type; + + // mask of elements that this processor is interested in + const unsigned int m_interests; + + // constructor for use by implementing classes only + geometry_processor(int srid, const std::string &type, unsigned int interests); +}; + + +//various bits for continuous processing of ways +struct way_helper +{ + way_helper(); + ~way_helper(); + size_t set(const idlist_t &node_ids, const middle_query_t *mid); + + nodelist_t node_cache; +}; + +//various bits for continuous processing of members of relations +struct relation_helper +{ + relation_helper(); + ~relation_helper(); + size_t set(const memberlist_t *member_list, const middle_t *mid); + + const memberlist_t *members; + multitaglist_t tags; + multinodelist_t nodes; + idlist_t ways; + rolelist_t roles; + std::vector superseeded; + +private: + idlist_t input_way_ids; +}; + +#endif /* GEOMETRY_PROCESSOR_HPP */ diff -Nru osm2pgsql-0.82.0/.gitignore osm2pgsql-0.88.1/.gitignore --- osm2pgsql-0.82.0/.gitignore 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/.gitignore 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,68 @@ +*.o +*~ +Makefile.in +aclocal.m4 +autom4te.cache/ +config.guess +config.h.in +config.h.in~ +config.sub +configure +compile +depcomp +fileformat.pb-c.c +fileformat.pb-c.h +install-sh +legacy/Makefile.in +ltmain.sh +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 +missing +nodecachefilereader +osmformat.pb-c.c +osmformat.pb-c.h +osm2pgsql + +Makefile +config.h +config.log +config.nice +config.status +legacy/.deps/ +legacy/Makefile +libtool +.deps/ +stamp-h1 +INSTALL + +log +test-driver +test-suite.log +tests/.dirstamp +tests/test-parse-xml2 +tests/test-middle-ram +tests/test-middle-pgsql +tests/test-middle-flat +tests/test-pgsql-escape +tests/test-parse-options +tests/test-output-multi-tags +tests/test-output-multi-line +tests/test-output-multi-line-storage +tests/test-output-multi-point +tests/test-output-multi-point-multi-table +tests/test-output-multi-polygon +tests/test-output-multi-poly-trivial +tests/test-output-pgsql +tests/test-output-pgsql-tablespace +tests/test-output-pgsql-z_order +tests/test-expire-tiles +tests/*.log +tests/*.trs +tests/*.flat.nodes.bin + +.libs/ +*.lo +libosm2pgsql.la diff -Nru osm2pgsql-0.82.0/id-tracker.cpp osm2pgsql-0.88.1/id-tracker.cpp --- osm2pgsql-0.82.0/id-tracker.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/id-tracker.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,178 @@ +#include "id-tracker.hpp" + +#include +#include +#include +#include + +#include + +#define BLOCK_BITS (16) +#define BLOCK_SIZE (1 << BLOCK_BITS) +#define BLOCK_MASK (BLOCK_SIZE - 1) + +namespace { +/* block used to be just a std::vector of fixed size. however, + * it seems there's significant overhead in exposing std::vector::iterator + * and so this is now a minimal re-implementation. + * + * each block is BLOCK_SIZE bits, stored as a vector of uint32_t elements. + */ +struct block { + block() : bits(BLOCK_SIZE >> 5, 0) {} + inline bool operator[](size_t i) const { return (bits[i >> 5] & (1 << (i & 0x1f))) > 0; } + //returns true if the value actually caused a bit to flip + inline bool set(size_t i, bool value) { + uint32_t &bit = bits[i >> 5]; + uint32_t old = bit; + uint32_t mask = 1 << (i & 0x1f); + //allow the bit to become 1 if not already + if (value) { + bit |= mask; + }//force the bit to 0 if its not already + else { + bit &= ~mask; + } + //did it actually change the value + return old != bit; + } + // find the next bit which is set, starting from an initial offset + // of start. this offset is a bit like an iterator, but not fully + // supporting iterator movement forwards and backwards. + // + // returns BLOCK_SIZE if a set bit isn't found + size_t next_set(size_t start) const { + uint32_t bit_i = start >> 5; + + while ((bit_i < (BLOCK_SIZE >> 5)) && (bits[bit_i] == 0)) { + ++bit_i; + } + + if (bit_i >= (BLOCK_SIZE >> 5)) { return BLOCK_SIZE; } + uint32_t bit = bits[bit_i]; + size_t idx = bit_i << 5; + while ((bit & 1) == 0) { ++idx; bit >>= 1; } + return idx; + } +private: + std::vector bits; +}; +} // anonymous namespace + +struct id_tracker::pimpl { + pimpl(); + ~pimpl(); + + bool get(osmid_t id) const; + bool set(osmid_t id, bool value); + osmid_t pop_min(); + + typedef std::map map_t; + map_t pending; + osmid_t old_id; + size_t count; + // a cache of the next starting point to search for in the block. + // this significantly speeds up pop_min() because it doesn't need + // to repeatedly search the beginning of the block each time. + boost::optional next_start; +}; + +bool id_tracker::pimpl::get(osmid_t id) const { + const osmid_t block = id >> BLOCK_BITS, offset = id & BLOCK_MASK; + map_t::const_iterator itr = pending.find(block); + bool result = false; + + if (itr != pending.end()) { + result = itr->second[offset]; + } + + return result; +} + +bool id_tracker::pimpl::set(osmid_t id, bool value) { + const osmid_t block = id >> BLOCK_BITS, offset = id & BLOCK_MASK; + bool flipped = pending[block].set(offset, value); + // a set may potentially invalidate a next_start, as the bit + // set might be before the position of next_start. + if (next_start) { next_start = boost::none; } + return flipped; +} + +// find the first element in a block set to true +osmid_t id_tracker::pimpl::pop_min() { + osmid_t id = max(); + + while (next_start || !pending.empty()) { + map_t::iterator itr = pending.begin(); + block &b = itr->second; + size_t start = next_start.get_value_or(0); + + size_t b_itr = b.next_set(start); + if (b_itr != BLOCK_SIZE) { + b.set(b_itr, false); + id = (itr->first << BLOCK_BITS) | b_itr; + next_start = b_itr; + break; + + } else { + // no elements in this block - might as well delete + // the whole thing. + pending.erase(itr); + // since next_start is relative to the current + // block, which is ceasing to exist, then we need to + // reset it. + next_start = boost::none; + } + } + + return id; +} + +id_tracker::pimpl::pimpl() + : pending(), old_id(min()), count(0), next_start(boost::none) { +} + +id_tracker::pimpl::~pimpl() { +} + +id_tracker::id_tracker(): impl() { + impl.reset(new pimpl()); +} + +id_tracker::~id_tracker() { +} + +void id_tracker::mark(osmid_t id) { + //setting returns true if the id wasn't already marked + impl->count += size_t(impl->set(id, true)); + //we've marked something so we need to be able to pop it + //the assert below will fail though if we've already popped + //some that were > id so we have to essentially reset to + //allow for more pops to take place + impl->old_id = min(); +} + +bool id_tracker::is_marked(osmid_t id) { + return impl->get(id); +} + +osmid_t id_tracker::pop_mark() { + osmid_t id = impl->pop_min(); + + assert((id > impl->old_id) || !id_tracker::is_valid(id)); + impl->old_id = id; + + //we just go rid of one (if there were some to get rid of) + if(impl->count > 0) + impl->count--; + + return id; +} + +size_t id_tracker::size() { return impl->count; } + +osmid_t id_tracker::last_returned() const { return impl->old_id; } + +bool id_tracker::is_valid(osmid_t id) { return id != max(); } +osmid_t id_tracker::max() { return std::numeric_limits::max(); } +osmid_t id_tracker::min() { return std::numeric_limits::min(); } diff -Nru osm2pgsql-0.82.0/id-tracker.hpp osm2pgsql-0.88.1/id-tracker.hpp --- osm2pgsql-0.82.0/id-tracker.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/id-tracker.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,27 @@ +#ifndef ID_TRACKER_HPP +#define ID_TRACKER_HPP + +#include "osmtypes.hpp" +#include +#include + +struct id_tracker : public boost::noncopyable { + id_tracker(); + ~id_tracker(); + + void mark(osmid_t id); + bool is_marked(osmid_t id); + osmid_t pop_mark(); + size_t size(); + osmid_t last_returned() const; + + static bool is_valid(osmid_t); + static osmid_t max(); + static osmid_t min(); + +private: + struct pimpl; + boost::scoped_ptr impl; +}; + +#endif /* ID_TRACKER_HPP */ diff -Nru osm2pgsql-0.82.0/input.c osm2pgsql-0.88.1/input.c --- osm2pgsql-0.82.0/input.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/input.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,221 +0,0 @@ -#define _FILE_OFFSET_BITS 64 -#define _LARGEFILE64_SOURCE - -#ifdef __MINGW_H -# include -#else -#include -#include -#include -#include -#include -#include -#include -#endif - -#include -#include - -#include "sanitizer.h" -#include "input.h" - -struct Input { - char *name; - enum { plainFile, gzipFile, bzip2File } type; - void *fileHandle; - /* needed by bzip2 when decompressing from multiple streams. other - decompressors must ignore it. */ - FILE *systemHandle; - int eof; - char buf[4096]; - int buf_ptr, buf_fill; -}; - -/* tries to re-open the bz stream at the next stream start. - returns 0 on success, -1 on failure. */ -int bzReOpen(struct Input *ctx, int *error) { - /* for copying out the last unused part of the block which - has an EOS token in it. needed for re-initialising the - next stream. */ - unsigned char unused[BZ_MAX_UNUSED]; - void *unused_tmp_ptr = NULL; - int nUnused, i; - - BZ2_bzReadGetUnused(error, (BZFILE *)(ctx->fileHandle), &unused_tmp_ptr, &nUnused); - if (*error != BZ_OK) return -1; - - /* when bzReadClose is called the unused buffer is deallocated, - so it needs to be copied somewhere safe first. */ - for (i = 0; i < nUnused; ++i) - unused[i] = ((unsigned char *)unused_tmp_ptr)[i]; - - BZ2_bzReadClose(error, (BZFILE *)(ctx->fileHandle)); - if (*error != BZ_OK) return -1; - - /* reassign the file handle */ - ctx->fileHandle = BZ2_bzReadOpen(error, ctx->systemHandle, 0, 0, unused, nUnused); - if (ctx->fileHandle == NULL || *error != BZ_OK) return -1; - - return 0; -} - -int readFile(void *context, char * buffer, int len) -{ - struct Input *ctx = context; - void *f = ctx->fileHandle; - int l = 0, error = 0; - - if (ctx->eof || (len == 0)) - return 0; - - switch(ctx->type) { - case plainFile: - l = read(*(int *)f, buffer, len); - if (l <= 0) ctx->eof = 1; - break; - case gzipFile: - l = gzread((gzFile)f, buffer, len); - if (l <= 0) ctx->eof = 1; - break; - case bzip2File: - l = BZ2_bzRead(&error, (BZFILE *)f, buffer, len); - - /* error codes BZ_OK and BZ_STREAM_END are both "OK", but the stream - end means the reader needs to be reset from the original handle. */ - if (error != BZ_OK) { - /* for stream errors, try re-opening the stream before admitting defeat. */ - if (error != BZ_STREAM_END || bzReOpen(ctx, &error) != 0) { - l = 0; - ctx->eof = 1; - } - } - break; - default: - fprintf(stderr, "Bad file type\n"); - break; - } - - if (l < 0) { - fprintf(stderr, "File reader received error %d (%d)\n", l, error); - l = 0; - } - - return l; -} - -char inputGetChar(void *context) -{ - struct Input *ctx = context; - - if (ctx->buf_ptr == ctx->buf_fill) { - ctx->buf_fill = readFile(context, &ctx->buf[0], sizeof(ctx->buf)); - ctx->buf_ptr = 0; - if (ctx->buf_fill == 0) - return 0; - if (ctx->buf_fill < 0) { - perror("Error while reading file"); - exit(1); - } - } - return ctx->buf[ctx->buf_ptr++]; -} - -int inputEof(void *context) -{ - return ((struct Input *)context)->eof; -} - - -void *inputOpen(const char *name) -{ - const char *ext = strrchr(name, '.'); - struct Input *ctx = malloc (sizeof(*ctx)); - - if (!ctx) - return NULL; - - memset(ctx, 0, sizeof(*ctx)); - - ctx->name = malloc(strlen(name) + 1); - if (ctx->name) strcpy(ctx->name, name); - - if (ext && !strcmp(ext, ".gz")) { - ctx->fileHandle = (void *)gzopen(name, "rb"); - ctx->type = gzipFile; - } else if (ext && !strcmp(ext, ".bz2")) { - int error = 0; - ctx->systemHandle = fopen(name, "rb"); - if (!ctx->systemHandle) { - fprintf(stderr, "error while opening file %s\n", name); - exit(10); - } - - ctx->fileHandle = (void *)BZ2_bzReadOpen(&error, ctx->systemHandle, 0, 0, NULL, 0); - ctx->type = bzip2File; - - } else { - int *pfd = malloc(sizeof(int)); - if (pfd) { - if (!strcmp(name, "-")) { - *pfd = STDIN_FILENO; - } else { - int flags = O_RDONLY; -#ifdef O_LARGEFILE - flags |= O_LARGEFILE; -#endif - *pfd = open(name, flags); - if (*pfd < 0) { - free(pfd); - pfd = NULL; - } - } - } - ctx->fileHandle = (void *)pfd; - ctx->type = plainFile; - } - if (!ctx->fileHandle) { - fprintf(stderr, "error while opening file %s\n", name); - exit(10); - } - ctx->buf_ptr = 0; - ctx->buf_fill = 0; - return (void *)ctx; -} - -int inputClose(void *context) -{ - struct Input *ctx = context; - void *f = ctx->fileHandle; - - switch(ctx->type) { - case plainFile: - close(*(int *)f); - free(f); - break; - case gzipFile: - gzclose((gzFile)f); - break; - case bzip2File: - BZ2_bzclose((BZFILE *)f); - break; - default: - fprintf(stderr, "Bad file type\n"); - break; - } - - free(ctx->name); - free(ctx); - return 0; -} - -xmlTextReaderPtr inputUTF8(const char *name) -{ - void *ctx = inputOpen(name); - - if (!ctx) { - fprintf(stderr, "Input reader create failed for: %s\n", name); - return NULL; - } - - return xmlReaderForIO(readFile, inputClose, (void *)ctx, NULL, NULL, 0); -} diff -Nru osm2pgsql-0.82.0/input.cpp osm2pgsql-0.88.1/input.cpp --- osm2pgsql-0.82.0/input.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/input.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,226 @@ +#define _FILE_OFFSET_BITS 64 +#define _LARGEFILE64_SOURCE + +#ifdef _WIN32 +#include +#include +#define STDIN_FILENO 0 +#else +#include +#endif + +#include +#include +#include +#include +#include + +#include "input.hpp" + +struct Input { + char *name; + enum { plainFile, gzipFile, bzip2File } type; + void *fileHandle; + /* needed by bzip2 when decompressing from multiple streams. other + decompressors must ignore it. */ + FILE *systemHandle; + int eof; + char buf[4096]; + int buf_ptr, buf_fill; +}; + +/* tries to re-open the bz stream at the next stream start. + returns 0 on success, -1 on failure. */ +int bzReOpen(struct Input *ctx, int *error) { + /* for copying out the last unused part of the block which + has an EOS token in it. needed for re-initialising the + next stream. */ + unsigned char unused[BZ_MAX_UNUSED]; + void *unused_tmp_ptr = NULL; + int nUnused, i; + + BZ2_bzReadGetUnused(error, (BZFILE *)(ctx->fileHandle), &unused_tmp_ptr, &nUnused); + if (*error != BZ_OK) return -1; + + /* when bzReadClose is called the unused buffer is deallocated, + so it needs to be copied somewhere safe first. */ + for (i = 0; i < nUnused; ++i) + unused[i] = ((unsigned char *)unused_tmp_ptr)[i]; + + BZ2_bzReadClose(error, (BZFILE *)(ctx->fileHandle)); + if (*error != BZ_OK) return -1; + + /* reassign the file handle */ + ctx->fileHandle = BZ2_bzReadOpen(error, ctx->systemHandle, 0, 0, unused, nUnused); + if (ctx->fileHandle == NULL || *error != BZ_OK) return -1; + + return 0; +} + +int readFile(struct Input *ctx, char * buffer, int len) +{ + void *f = ctx->fileHandle; + int l = 0, error = 0; + + if (ctx->eof || (len == 0)) + return 0; + + switch(ctx->type) { + case Input::plainFile: + l = read(*(int *)f, buffer, len); + if (l <= 0) ctx->eof = 1; + break; + case Input::gzipFile: + l = gzread((gzFile)f, buffer, len); + if (l <= 0) ctx->eof = 1; + break; + case Input::bzip2File: + l = BZ2_bzRead(&error, (BZFILE *)f, buffer, len); + + /* error codes BZ_OK and BZ_STREAM_END are both "OK", but the stream + end means the reader needs to be reset from the original handle. */ + if (error != BZ_OK) { + /* for stream errors, try re-opening the stream before admitting defeat. */ + if (error != BZ_STREAM_END || bzReOpen(ctx, &error) != 0) { + l = 0; + ctx->eof = 1; + } + } + break; + default: + fprintf(stderr, "Bad file type\n"); + break; + } + + if (l < 0) { + fprintf(stderr, "File reader received error %d (%d)\n", l, error); + l = 0; + } + + return l; +} + +char inputGetChar(struct Input *ctx) +{ + if (ctx->buf_ptr == ctx->buf_fill) { + ctx->buf_fill = readFile(ctx, &ctx->buf[0], sizeof(ctx->buf)); + ctx->buf_ptr = 0; + if (ctx->buf_fill == 0) + return 0; + if (ctx->buf_fill < 0) { + perror("Error while reading file"); + exit(1); + } + } + return ctx->buf[ctx->buf_ptr++]; +} + +int inputEof(struct Input *ctx) +{ + return ctx->eof; +} + + +struct Input *inputOpen(const char *name) +{ + const char *ext = strrchr(name, '.'); + struct Input *ctx = (struct Input *)malloc(sizeof(*ctx)); + + if (!ctx) + return NULL; + + memset(ctx, 0, sizeof(*ctx)); + + ctx->name = (char *)malloc(strlen(name) + 1); + if (ctx->name) strcpy(ctx->name, name); + + if (ext && !strcmp(ext, ".gz")) { + ctx->fileHandle = (void *)gzopen(name, "rb"); + ctx->type = Input::gzipFile; + } else if (ext && !strcmp(ext, ".bz2")) { + int error = 0; + ctx->systemHandle = fopen(name, "rb"); + if (!ctx->systemHandle) { + fprintf(stderr, "error while opening file %s\n", name); + exit(10); + } + + ctx->fileHandle = (void *)BZ2_bzReadOpen(&error, ctx->systemHandle, 0, 0, NULL, 0); + ctx->type = Input::bzip2File; + + } else { + int *pfd = (int *)malloc(sizeof(int)); + if (pfd) { + if (!strcmp(name, "-")) { + *pfd = STDIN_FILENO; + } else { + int flags = O_RDONLY; +#ifdef O_LARGEFILE + flags |= O_LARGEFILE; +#endif + *pfd = open(name, flags); + if (*pfd < 0) { + free(pfd); + pfd = NULL; + } + } + } + ctx->fileHandle = (void *)pfd; + ctx->type = Input::plainFile; + } + if (!ctx->fileHandle) { + fprintf(stderr, "error while opening file %s\n", name); + exit(10); + } + ctx->buf_ptr = 0; + ctx->buf_fill = 0; + return ctx; +} + +int inputClose(struct Input *ctx) +{ + void *f = ctx->fileHandle; + + switch(ctx->type) { + case Input::plainFile: + close(*(int *)f); + free(f); + break; + case Input::gzipFile: + gzclose((gzFile)f); + break; + case Input::bzip2File: + BZ2_bzclose((BZFILE *)f); + break; + default: + fprintf(stderr, "Bad file type\n"); + break; + } + + free(ctx->name); + free(ctx); + return 0; +} + +// NOTE: need the two function variants below to fit into what +// libXML expects the function signatures to be. + +static int readFileXML(void *context, char *buffer, int len) { + return readFile((struct Input *)context, buffer, len); +} + +static int inputCloseXML(void *context) { + return inputClose((struct Input *)context); +} + +xmlTextReaderPtr inputUTF8(const char *name) +{ + void *ctx = inputOpen(name); + + if (!ctx) { + fprintf(stderr, "Input reader create failed for: %s\n", name); + return NULL; + } + + return xmlReaderForIO(readFileXML, inputCloseXML, (void *)ctx, NULL, NULL, 0); +} diff -Nru osm2pgsql-0.82.0/input.h osm2pgsql-0.88.1/input.h --- osm2pgsql-0.82.0/input.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/input.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -#ifndef INPUT_H -#define INPUT_H - -int readFile(void *context, char * buffer, int len); -int inputClose(void *context); -void *inputOpen(const char *name); -char inputGetChar(void *context); -int inputEof(void *context); -xmlTextReaderPtr inputUTF8(const char *name); - -#endif diff -Nru osm2pgsql-0.82.0/input.hpp osm2pgsql-0.88.1/input.hpp --- osm2pgsql-0.82.0/input.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/input.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,14 @@ +#ifndef INPUT_H +#define INPUT_H + +#include +struct Input; + +int readFile(struct Input *context, char * buffer, int len); +int inputClose(struct Input *context); +struct Input *inputOpen(const char *name); +char inputGetChar(struct Input *context); +int inputEof(struct Input *context); +xmlTextReaderPtr inputUTF8(const char *name); + +#endif diff -Nru osm2pgsql-0.82.0/INSTALL osm2pgsql-0.88.1/INSTALL --- osm2pgsql-0.82.0/INSTALL 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/INSTALL 1970-01-01 00:00:00.000000000 +0000 @@ -1,365 +0,0 @@ -Installation Instructions -************************* - -Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005, -2006, 2007, 2008, 2009 Free Software Foundation, Inc. - - Copying and distribution of this file, with or without modification, -are permitted in any medium without royalty provided the copyright -notice and this notice are preserved. This file is offered as-is, -without warranty of any kind. - -Basic Installation -================== - - Briefly, the shell commands `./configure; make; make install' should -configure, build, and install this package. The following -more-detailed instructions are generic; see the `README' file for -instructions specific to this package. Some packages provide this -`INSTALL' file but do not implement all of the features documented -below. The lack of an optional feature in a given package is not -necessarily a bug. More recommendations for GNU packages can be found -in *note Makefile Conventions: (standards)Makefile Conventions. - - The `configure' shell script attempts to guess correct values for -various system-dependent variables used during compilation. It uses -those values to create a `Makefile' in each directory of the package. -It may also create one or more `.h' files containing system-dependent -definitions. Finally, it creates a shell script `config.status' that -you can run in the future to recreate the current configuration, and a -file `config.log' containing compiler output (useful mainly for -debugging `configure'). - - It can also use an optional file (typically called `config.cache' -and enabled with `--cache-file=config.cache' or simply `-C') that saves -the results of its tests to speed up reconfiguring. Caching is -disabled by default to prevent problems with accidental use of stale -cache files. - - If you need to do unusual things to compile the package, please try -to figure out how `configure' could check whether to do them, and mail -diffs or instructions to the address given in the `README' so they can -be considered for the next release. If you are using the cache, and at -some point `config.cache' contains results you don't want to keep, you -may remove or edit it. - - The file `configure.ac' (or `configure.in') is used to create -`configure' by a program called `autoconf'. You need `configure.ac' if -you want to change it or regenerate `configure' using a newer version -of `autoconf'. - - The simplest way to compile this package is: - - 1. `cd' to the directory containing the package's source code and type - `./configure' to configure the package for your system. - - Running `configure' might take a while. While running, it prints - some messages telling which features it is checking for. - - 2. Type `make' to compile the package. - - 3. Optionally, type `make check' to run any self-tests that come with - the package, generally using the just-built uninstalled binaries. - - 4. Type `make install' to install the programs and any data files and - documentation. When installing into a prefix owned by root, it is - recommended that the package be configured and built as a regular - user, and only the `make install' phase executed with root - privileges. - - 5. Optionally, type `make installcheck' to repeat any self-tests, but - this time using the binaries in their final installed location. - This target does not install anything. Running this target as a - regular user, particularly if the prior `make install' required - root privileges, verifies that the installation completed - correctly. - - 6. You can remove the program binaries and object files from the - source code directory by typing `make clean'. To also remove the - files that `configure' created (so you can compile the package for - a different kind of computer), type `make distclean'. There is - also a `make maintainer-clean' target, but that is intended mainly - for the package's developers. If you use it, you may have to get - all sorts of other programs in order to regenerate files that came - with the distribution. - - 7. Often, you can also type `make uninstall' to remove the installed - files again. In practice, not all packages have tested that - uninstallation works correctly, even though it is required by the - GNU Coding Standards. - - 8. Some packages, particularly those that use Automake, provide `make - distcheck', which can by used by developers to test that all other - targets like `make install' and `make uninstall' work correctly. - This target is generally not run by end users. - -Compilers and Options -===================== - - Some systems require unusual options for compilation or linking that -the `configure' script does not know about. Run `./configure --help' -for details on some of the pertinent environment variables. - - You can give `configure' initial values for configuration parameters -by setting variables in the command line or in the environment. Here -is an example: - - ./configure CC=c99 CFLAGS=-g LIBS=-lposix - - *Note Defining Variables::, for more details. - -Compiling For Multiple Architectures -==================================== - - You can compile the package for more than one kind of computer at the -same time, by placing the object files for each architecture in their -own directory. To do this, you can use GNU `make'. `cd' to the -directory where you want the object files and executables to go and run -the `configure' script. `configure' automatically checks for the -source code in the directory that `configure' is in and in `..'. This -is known as a "VPATH" build. - - With a non-GNU `make', it is safer to compile the package for one -architecture at a time in the source code directory. After you have -installed the package for one architecture, use `make distclean' before -reconfiguring for another architecture. - - On MacOS X 10.5 and later systems, you can create libraries and -executables that work on multiple system types--known as "fat" or -"universal" binaries--by specifying multiple `-arch' options to the -compiler but only a single `-arch' option to the preprocessor. Like -this: - - ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CPP="gcc -E" CXXCPP="g++ -E" - - This is not guaranteed to produce working output in all cases, you -may have to build one architecture at a time and combine the results -using the `lipo' tool if you have problems. - -Installation Names -================== - - By default, `make install' installs the package's commands under -`/usr/local/bin', include files under `/usr/local/include', etc. You -can specify an installation prefix other than `/usr/local' by giving -`configure' the option `--prefix=PREFIX', where PREFIX must be an -absolute file name. - - You can specify separate installation prefixes for -architecture-specific files and architecture-independent files. If you -pass the option `--exec-prefix=PREFIX' to `configure', the package uses -PREFIX as the prefix for installing programs and libraries. -Documentation and other data files still use the regular prefix. - - In addition, if you use an unusual directory layout you can give -options like `--bindir=DIR' to specify different values for particular -kinds of files. Run `configure --help' for a list of the directories -you can set and what kinds of files go in them. In general, the -default for these options is expressed in terms of `${prefix}', so that -specifying just `--prefix' will affect all of the other directory -specifications that were not explicitly provided. - - The most portable way to affect installation locations is to pass the -correct locations to `configure'; however, many packages provide one or -both of the following shortcuts of passing variable assignments to the -`make install' command line to change installation locations without -having to reconfigure or recompile. - - The first method involves providing an override variable for each -affected directory. For example, `make install -prefix=/alternate/directory' will choose an alternate location for all -directory configuration variables that were expressed in terms of -`${prefix}'. Any directories that were specified during `configure', -but not in terms of `${prefix}', must each be overridden at install -time for the entire installation to be relocated. The approach of -makefile variable overrides for each directory variable is required by -the GNU Coding Standards, and ideally causes no recompilation. -However, some platforms have known limitations with the semantics of -shared libraries that end up requiring recompilation when using this -method, particularly noticeable in packages that use GNU Libtool. - - The second method involves providing the `DESTDIR' variable. For -example, `make install DESTDIR=/alternate/directory' will prepend -`/alternate/directory' before all installation names. The approach of -`DESTDIR' overrides is not required by the GNU Coding Standards, and -does not work on platforms that have drive letters. On the other hand, -it does better at avoiding recompilation issues, and works well even -when some directory options were not specified in terms of `${prefix}' -at `configure' time. - -Optional Features -================= - - If the package supports it, you can cause programs to be installed -with an extra prefix or suffix on their names by giving `configure' the -option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. - - Some packages pay attention to `--enable-FEATURE' options to -`configure', where FEATURE indicates an optional part of the package. -They may also pay attention to `--with-PACKAGE' options, where PACKAGE -is something like `gnu-as' or `x' (for the X Window System). The -`README' should mention any `--enable-' and `--with-' options that the -package recognizes. - - For packages that use the X Window System, `configure' can usually -find the X include and library files automatically, but if it doesn't, -you can use the `configure' options `--x-includes=DIR' and -`--x-libraries=DIR' to specify their locations. - - Some packages offer the ability to configure how verbose the -execution of `make' will be. For these packages, running `./configure ---enable-silent-rules' sets the default to minimal output, which can be -overridden with `make V=1'; while running `./configure ---disable-silent-rules' sets the default to verbose, which can be -overridden with `make V=0'. - -Particular systems -================== - - On HP-UX, the default C compiler is not ANSI C compatible. If GNU -CC is not installed, it is recommended to use the following options in -order to use an ANSI C compiler: - - ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" - -and if that doesn't work, install pre-built binaries of GCC for HP-UX. - - On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot -parse its `' header file. The option `-nodtk' can be used as -a workaround. If GNU CC is not installed, it is therefore recommended -to try - - ./configure CC="cc" - -and if that doesn't work, try - - ./configure CC="cc -nodtk" - - On Solaris, don't put `/usr/ucb' early in your `PATH'. This -directory contains several dysfunctional programs; working variants of -these programs are available in `/usr/bin'. So, if you need `/usr/ucb' -in your `PATH', put it _after_ `/usr/bin'. - - On Haiku, software installed for all users goes in `/boot/common', -not `/usr/local'. It is recommended to use the following options: - - ./configure --prefix=/boot/common - -Specifying the System Type -========================== - - There may be some features `configure' cannot figure out -automatically, but needs to determine by the type of machine the package -will run on. Usually, assuming the package is built to be run on the -_same_ architectures, `configure' can figure that out, but if it prints -a message saying it cannot guess the machine type, give it the -`--build=TYPE' option. TYPE can either be a short name for the system -type, such as `sun4', or a canonical name which has the form: - - CPU-COMPANY-SYSTEM - -where SYSTEM can have one of these forms: - - OS - KERNEL-OS - - See the file `config.sub' for the possible values of each field. If -`config.sub' isn't included in this package, then this package doesn't -need to know the machine type. - - If you are _building_ compiler tools for cross-compiling, you should -use the option `--target=TYPE' to select the type of system they will -produce code for. - - If you want to _use_ a cross compiler, that generates code for a -platform different from the build platform, you should specify the -"host" platform (i.e., that on which the generated programs will -eventually be run) with `--host=TYPE'. - -Sharing Defaults -================ - - If you want to set default values for `configure' scripts to share, -you can create a site shell script called `config.site' that gives -default values for variables like `CC', `cache_file', and `prefix'. -`configure' looks for `PREFIX/share/config.site' if it exists, then -`PREFIX/etc/config.site' if it exists. Or, you can set the -`CONFIG_SITE' environment variable to the location of the site script. -A warning: not all `configure' scripts look for a site script. - -Defining Variables -================== - - Variables not defined in a site shell script can be set in the -environment passed to `configure'. However, some packages may run -configure again during the build, and the customized values of these -variables may be lost. In order to avoid this problem, you should set -them in the `configure' command line, using `VAR=value'. For example: - - ./configure CC=/usr/local2/bin/gcc - -causes the specified `gcc' to be used as the C compiler (unless it is -overridden in the site shell script). - -Unfortunately, this technique does not work for `CONFIG_SHELL' due to -an Autoconf bug. Until the bug is fixed you can use this workaround: - - CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash - -`configure' Invocation -====================== - - `configure' recognizes the following options to control how it -operates. - -`--help' -`-h' - Print a summary of all of the options to `configure', and exit. - -`--help=short' -`--help=recursive' - Print a summary of the options unique to this package's - `configure', and exit. The `short' variant lists options used - only in the top level, while the `recursive' variant lists options - also present in any nested packages. - -`--version' -`-V' - Print the version of Autoconf used to generate the `configure' - script, and exit. - -`--cache-file=FILE' - Enable the cache: use and save the results of the tests in FILE, - traditionally `config.cache'. FILE defaults to `/dev/null' to - disable caching. - -`--config-cache' -`-C' - Alias for `--cache-file=config.cache'. - -`--quiet' -`--silent' -`-q' - Do not print messages saying which checks are being made. To - suppress all normal output, redirect it to `/dev/null' (any error - messages will still be shown). - -`--srcdir=DIR' - Look for the package's source code in directory DIR. Usually - `configure' can determine that directory automatically. - -`--prefix=DIR' - Use DIR as the installation prefix. *note Installation Names:: - for more details, including other options available for fine-tuning - the installation locations. - -`--no-create' -`-n' - Run the configure checks, but stop before creating any output - files. - -`configure' also accepts some other, not widely useful, options. Run -`configure --help' for more details. - diff -Nru osm2pgsql-0.82.0/install-postgis-osm-db.sh osm2pgsql-0.88.1/install-postgis-osm-db.sh --- osm2pgsql-0.82.0/install-postgis-osm-db.sh 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/install-postgis-osm-db.sh 2015-08-15 05:28:11.000000000 +0000 @@ -21,7 +21,12 @@ sudo -u postgres createlang plpgsql $DBNAME || true - if [ -e /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql ] ; then + if [ -e /usr/share/postgresql/9.3/extension/postgis.control ]; then + echo "Initializing Spatial Extentions for postgresql 9.3" + echo "CREATE EXTENSION postgis;" | sudo -u postgres psql $DBNAME + echo "Initializing hstore" + echo "CREATE EXTENSION hstore;" | sudo -u postgres psql $DBNAME + else if [ -e /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql ] ; then echo "Initializing Spatial Extentions for postgresql 9.1" file_postgis=/usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql file_spatial_ref=/usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql @@ -44,7 +49,7 @@ echo "Initializing hstore" file_hstore=/usr/share/postgresql/8.4/contrib/hstore.sql sudo -u postgres psql $DBNAME <$file_hstore >/dev/null 2>&1 - fi + fi fi echo "Setting ownership to user $DBOWNER" diff -Nru osm2pgsql-0.82.0/keyvals.c osm2pgsql-0.88.1/keyvals.c --- osm2pgsql-0.82.0/keyvals.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/keyvals.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,346 +0,0 @@ -/* Common key-value list processing - * - * Used as a small general purpose store for - * tags, segment lists etc - * - */ -#define USE_TREE - -#include -#include -#include -#include -#include -#include "keyvals.h" - -#ifdef USE_TREE -#include "text-tree.h" -#endif - -void initList(struct keyval *head) -{ - assert(head); - - head->next = head; - head->prev = head; - head->key = NULL; - head->value = NULL; - head->has_column = 0; -} - -void freeItem(struct keyval *p) -{ - if (!p) - return; - -#ifdef USE_TREE - text_release(tree_ctx, p->key); - text_release(tree_ctx, p->value); -#else - free(p->key); - free(p->value); -#endif - free(p); -} - - -unsigned int countList(struct keyval *head) -{ - struct keyval *p; - unsigned int count = 0; - - if (!head) - return 0; - - p = head->next; - while(p != head) { - count++; - p = p->next; - } - return count; -} - -int listHasData(struct keyval *head) -{ - if (!head) - return 0; - - return (head->next != head); -} - - -char *getItem(struct keyval *head, const char *name) -{ - struct keyval *p; - - if (!head) - return NULL; - - p = head->next; - while(p != head) { - if (!strcmp(p->key, name)) - return p->value; - p = p->next; - } - return NULL; -} - -/* unlike getItem this function gives a pointer to the whole - list item which can be used to remove the tag from the linked list - with the removeTag function -*/ -struct keyval *getTag(struct keyval *head, const char *name) -{ - struct keyval *p; - - if (!head) - return NULL; - - p = head->next; - while(p != head) { - if (!strcmp(p->key, name)) - return p; - p = p->next; - } - return NULL; -} - -void removeTag(struct keyval *tag) -{ - tag->prev->next=tag->next; - tag->next->prev=tag->prev; - freeItem(tag); -} - -struct keyval *firstItem(struct keyval *head) -{ - if (head == NULL || head == head->next) - return NULL; - - return head->next; -} - -struct keyval *nextItem(struct keyval *head, struct keyval *item) -{ - if (item->next == head) - return NULL; - - return item->next; -} - -/* Pulls all items from list which match this prefix - * note: they are removed from the original list an returned in a new one - */ -struct keyval *getMatches(struct keyval *head, const char *name) -{ - struct keyval *out = NULL; - struct keyval *p; - - if (!head) - return NULL; - - out = malloc(sizeof(struct keyval)); - if (!out) - return NULL; - - initList(out); - p = head->next; - while(p != head) { - struct keyval *next = p->next; - if (!strncmp(p->key, name, strlen(name))) { - p->next->prev = p->prev; - p->prev->next = p->next; - pushItem(out, p); - } - p = next; - } - - if (listHasData(out)) - return out; - - free(out); - return NULL; -} - -void updateItem(struct keyval *head, const char *name, const char *value) -{ - struct keyval *item; - - if (!head) - return; - - item = head->next; - while(item != head) { - if (!strcmp(item->key, name)) { -#ifdef USE_TREE - text_release(tree_ctx, item->value); - item->value = (char *)text_get(tree_ctx,value); -#else - free(item->value); - item->value = strdup(value); -#endif - return; - } - item = item->next; - } - addItem(head, name, value, 0); -} - - -struct keyval *popItem(struct keyval *head) -{ - struct keyval *p; - - if (!head) - return NULL; - - p = head->next; - if (p == head) - return NULL; - - head->next = p->next; - p->next->prev = head; - - p->next = NULL; - p->prev = NULL; - - return p; -} - - -void pushItem(struct keyval *head, struct keyval *item) -{ - - assert(head); - assert(item); - - item->next = head; - item->prev = head->prev; - head->prev->next = item; - head->prev = item; -} - -int addItem(struct keyval *head, const char *name, const char *value, int noDupe) -{ - struct keyval *item; - - assert(head); - assert(name); - assert(value); - - if (noDupe) { - item = head->next; - while (item != head) { - if (!strcmp(item->value, value) && !strcmp(item->key, name)) - return 1; - item = item->next; - } - } - - item = malloc(sizeof(struct keyval)); - - if (!item) { - fprintf(stderr, "Error allocating keyval\n"); - return 2; - } - -#ifdef USE_TREE - item->key = (char *)text_get(tree_ctx,name); - item->value = (char *)text_get(tree_ctx,value); -#else - item->key = strdup(name); - item->value = strdup(value); -#endif - item->has_column=0; - - -#if 1 - /* Add to head */ - item->next = head->next; - item->prev = head; - head->next->prev = item; - head->next = item; -#else - /* Add to tail */ - item->prev = head->prev; - item->next = head; - head->prev->next = item; - head->prev = item; -#endif - return 0; -} - -void resetList(struct keyval *head) -{ - struct keyval *item; - - while((item = popItem(head))) - freeItem(item); -} - -void cloneList( struct keyval *target, struct keyval *source ) -{ - struct keyval *ptr; - for( ptr = source->next; ptr != source; ptr=ptr->next ) - addItem( target, ptr->key, ptr->value, 0 ); -} - - -/* create an escaped version of the string for hstore table insert */ -/* make shure dst is 2*strlen(src) */ -static void escape4hstore(char *dst, char *src) { - size_t i,j; - - j=0; - for (i=0;ikey, tags->value); -} - -void keyval2hstore_manual(char *hstring, char *key, char *value) -{ - static char* str=NULL; - static size_t stlen=0; - size_t len; - - len=strlen(value); - if (len>stlen) { - stlen=len; - str=realloc(str,1+stlen*2); - } - - len=strlen(key); - if (len>stlen) { - stlen=len; - str=realloc(str,1+stlen*2); - } - - escape4hstore(str,key); - hstring+=sprintf(hstring,"\"%s\"=>",str); - escape4hstore(str,value); - sprintf(hstring,"\"%s\"",str); -} - diff -Nru osm2pgsql-0.82.0/keyvals.h osm2pgsql-0.88.1/keyvals.h --- osm2pgsql-0.82.0/keyvals.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/keyvals.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,41 +0,0 @@ -/* Common key-value list processing - * - * Used as a small general purpose store for - * tags, segment lists etc - * - */ - -#ifndef KEYVAL_H -#define KEYVAL_H - -struct keyval { - char *key; - char *value; - /* if a hstore column is requested we need a flag to store if a key - has its own column because it should not be added to the hstore - in this case - */ - int has_column; - struct keyval *next; - struct keyval *prev; -}; - -void initList(struct keyval *head); -void freeItem(struct keyval *p); -unsigned int countList(struct keyval *head); -int listHasData(struct keyval *head); -char *getItem(struct keyval *head, const char *name); -struct keyval *getTag(struct keyval *head, const char *name); -void removeTag(struct keyval *tag); -struct keyval *firstItem(struct keyval *head); -struct keyval *nextItem(struct keyval *head, struct keyval *item); -struct keyval *popItem(struct keyval *head); -void pushItem(struct keyval *head, struct keyval *item); -int addItem(struct keyval *head, const char *name, const char *value, int noDupe); -void resetList(struct keyval *head); -struct keyval *getMatches(struct keyval *head, const char *name); -void updateItem(struct keyval *head, const char *name, const char *value); -void cloneList( struct keyval *target, struct keyval *source ); -void keyval2hstore(char *hstring, struct keyval *tags); -void keyval2hstore_manual(char *hstring, char *key, char *value); -#endif diff -Nru osm2pgsql-0.82.0/legacy/build_geometry.cpp osm2pgsql-0.88.1/legacy/build_geometry.cpp --- osm2pgsql-0.82.0/legacy/build_geometry.cpp 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/legacy/build_geometry.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,163 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#include -#include - -#if (GEOS_VERSION_MAJOR==3) -/* geos trunk (3.0.0rc) */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace geos::geom; -using namespace geos::io; -using namespace geos::operation::linemerge; -#else -/* geos-2.2 */ -#include -#include -#include -using namespace geos; -#endif - -#include "build_geometry.h" - - -struct Segment -{ - Segment(double x0_,double y0_,double x1_,double y1_) - :x0(x0_),y0(y0_),x1(x1_),y1(y1_) {} - - double x0; - double y0; - double x1; - double y1; -}; - -static std::vector segs; -static std::vector wkts; - -typedef std::auto_ptr geom_ptr; - -int is_simple(const char* wkt) -{ - GeometryFactory factory; - WKTReader reader(&factory); - geom_ptr geom(reader.read(wkt)); - if (geom->isSimple()) return 1; - return 0; -} - -void add_segment(double x0,double y0,double x1,double y1) -{ - segs.push_back(Segment(x0,y0,x1,y1)); -} - -const char * get_wkt(size_t index) -{ - return wkts[index].c_str(); -} - -void clear_wkts() -{ - wkts.clear(); -} - -size_t build_geometry(int polygon) -{ - size_t wkt_size = 0; - GeometryFactory factory; - geom_ptr segment(0); - std::auto_ptr > lines(new std::vector); - std::vector::const_iterator pos=segs.begin(); - std::vector::const_iterator end=segs.end(); - bool first=true; - try { - while (pos != end) - { - if (pos->x0 != pos->x1 || pos->y0 != pos->y1) - { - std::auto_ptr coords(factory.getCoordinateSequenceFactory()->create(0,2)); - coords->add(Coordinate(pos->x0,pos->y0)); - coords->add(Coordinate(pos->x1,pos->y1)); - geom_ptr linestring(factory.createLineString(coords.release())); - if (first) - { - segment = linestring; - first=false; - } - else - { - lines->push_back(linestring.release()); - } - } - ++pos; - } - - segs.clear(); - - if (segment.get()) - { - geom_ptr mline (factory.createMultiLineString(lines.release())); - geom_ptr noded (segment->Union(mline.get())); - LineMerger merger; - merger.add(noded.get()); - std::auto_ptr > merged(merger.getMergedLineStrings()); - WKTWriter writer; - - for (unsigned i=0 ;i < merged->size(); ++i) - { - std::auto_ptr pline ((*merged ) [i]); - - if (polygon == 1 && pline->getNumPoints() > 3 && pline->isClosed()) - { - std::auto_ptr ring(factory.createLinearRing(pline->getCoordinates())); - geom_ptr poly(factory.createPolygon(ring.release(),0)); - std::string text = writer.write(poly.get()); - - wkts.push_back(text); - ++wkt_size; - } - else - { - std::string text = writer.write(pline.get()); - wkts.push_back(text); - ++wkt_size; - } - } - } - } - catch (...) - { - std::cerr << "excepton caught \n"; - wkt_size = 0; - } - return wkt_size; -} - diff -Nru osm2pgsql-0.82.0/legacy/build_geometry.h osm2pgsql-0.88.1/legacy/build_geometry.h --- osm2pgsql-0.82.0/legacy/build_geometry.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/legacy/build_geometry.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,40 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# Part of osm2pgsql utility -#----------------------------------------------------------------------------- -# By Artem Pavlenko, Copyright 2007 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef BUILD_GEOMETRY_H -#define BUILD_GEOMETRY_H - -#ifdef __cplusplus -extern "C" { -#endif - -int is_simple(const char* wkt); -void add_segment(double x0,double y0,double x1, double y1); -const char* get_wkt(size_t index); -size_t build_geometry(int polygon); -void clear_wkts(); - -#ifdef __cplusplus -} -#endif - -#endif diff -Nru osm2pgsql-0.82.0/legacy/Makefile.am osm2pgsql-0.88.1/legacy/Makefile.am --- osm2pgsql-0.82.0/legacy/Makefile.am 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/legacy/Makefile.am 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -bin_PROGRAMS = osm2pgsql-legacy - -osm2pgsql_legacy_SOURCES = build_geometry.cpp osm2pgsql.c - -AM_CFLAGS=@XML2_CFLAGS@ @GEOS_CFLAGS@ -AM_CPPFLAGS=@XML2_CFLAGS@ @GEOS_CFLAGS@ - -AM_LDFLAGS=@XML2_LDFLAGS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ - -noinst_HEADERS = build_geometry.h - diff -Nru osm2pgsql-0.82.0/legacy/osm2pgsql.c osm2pgsql-0.88.1/legacy/osm2pgsql.c --- osm2pgsql-0.82.0/legacy/osm2pgsql.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/legacy/osm2pgsql.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,642 +0,0 @@ -/* - #----------------------------------------------------------------------------- - # osm2pgsql - converts planet.osm file into PostgreSQL - # compatible output suitable to be rendered by mapnik - # Use: osm2pgsql planet.osm > planet.sql - #----------------------------------------------------------------------------- - # Original Python implementation by Artem Pavlenko - # Re-implementation by Jon Burgess, Copyright 2006 - # - # This program is free software; you can redistribute it and/or - # modify it under the terms of the GNU General Public License - # as published by the Free Software Foundation; either version 2 - # of the License, or (at your option) any later version. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - # GNU General Public License for more details. - # - # You should have received a copy of the GNU General Public License - # along with this program; if not, write to the Free Software - # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - #----------------------------------------------------------------------------- -*/ - -#define _GNU_SOURCE - -#include -#include -#include -#include -#include - -#include -#include - -#include "build_geometry.h" - -#if 0 -#define DEBUG printf -#else -#define DEBUG(x, ...) -#endif - -struct tagDesc { - const char *name; - const char *type; - const int polygon; -}; - -static struct tagDesc exportTags[] = { - {"name", "text", 0}, - {"place", "text", 0}, - {"landuse", "text", 1}, - {"leisure", "text", 1}, - {"natural", "text", 1}, - {"man_made","text", 0}, - {"waterway","text", 0}, - {"highway", "text", 0}, - {"foot", "text", 0}, - {"horse", "text", 0}, - {"bicycle", "text", 0}, - {"motorcar","text", 0}, - {"residence","text", 0}, - {"railway", "text", 0}, - {"amenity", "text", 1}, - {"tourism", "text", 1}, - {"learning","text", 0}, - {"building","text", 1}, - {"bridge", "text", 0}, - {"layer", "text", 0}, - {"junction","text", 0}, - {"sport", "text", 1}, - {"route", "text", 0}, - {"aeroway", "text", 0} -}; - -static const char *table_name_point = "planet_osm_point"; -static const char *table_name_line = "planet_osm_line"; -static const char *table_name_polygon = "planet_osm_polygon"; - -#define MAX_ID_NODE (35000000) -#define MAX_ID_SEGMENT (35000000) - -struct osmNode { - double lon; - double lat; -}; - -struct osmSegment { - unsigned int from; - unsigned int to; -}; - -struct osmWay { - char *values; - char *wkt; -}; - -static struct osmNode nodes[MAX_ID_NODE+1]; -static struct osmSegment segments[MAX_ID_SEGMENT+1]; - -static int count_node, count_all_node, max_node; -static int count_segment, count_all_segment, max_segment; -static int count_way, count_all_way, max_way; -static int count_way_seg; - -struct keyval { - char *key; - char *value; - struct keyval *next; - struct keyval *prev; -}; - - -static struct keyval keys, tags, segs; - - -void usage(const char *arg0) -{ - fprintf(stderr, "Usage error:\n\t%s planet.osm > planet.sql\n", arg0); - fprintf(stderr, "or\n\tgzip -dc planet.osm.gz | %s - | gzip -c > planet.sql.gz\n", arg0); -} - -void initList(struct keyval *head) -{ - head->next = head; - head->prev = head; - head->key = NULL; - head->value = NULL; -} - -void freeItem(struct keyval *p) -{ - free(p->key); - free(p->value); - free(p); -} - - -unsigned int countList(struct keyval *head) -{ - struct keyval *p = head->next; - unsigned int count = 0; - - while(p != head) { - count++; - p = p->next; - } - return count; -} - -int listHasData(struct keyval *head) -{ - return (head->next != head); -} - - -char *getItem(struct keyval *head, const char *name) -{ - struct keyval *p = head->next; - while(p != head) { - if (!strcmp(p->key, name)) - return p->value; - p = p->next; - } - return NULL; -} - - -struct keyval *popItem(struct keyval *head) -{ - struct keyval *p = head->next; - if (p == head) - return NULL; - - head->next = p->next; - p->next->prev = head; - - p->next = NULL; - p->prev = NULL; - - return p; -} - - -void pushItem(struct keyval *head, struct keyval *item) -{ - item->next = head; - item->prev = head->prev; - head->prev->next = item; - head->prev = item; -} - -int addItem(struct keyval *head, const char *name, const char *value, int noDupe) -{ - struct keyval *item; - - if (noDupe) { - item = head->next; - while (item != head) { - if (!strcmp(item->value, value) && !strcmp(item->key, name)) { - //fprintf(stderr, "Discarded %s=%s\n", name, value); - return 1; - } - item = item->next; - } - } - - item = malloc(sizeof(struct keyval)); - - if (!item) { - fprintf(stderr, "Error allocating keyval\n"); - return 2; - } - - item->key = strdup(name); - item->value = strdup(value); - - item->next = head->next; - item->prev = head; - head->next->prev = item; - head->next = item; - - return 0; -} - -void resetList(struct keyval *head) -{ - struct keyval *item; - - while((item = popItem(head))) - freeItem(item); -} - -size_t WKT(int polygon) -{ - while (listHasData(&segs)) - { - struct keyval *p; - unsigned int id, to, from; - double x0, y0, x1, y1; - p = popItem(&segs); - id = strtoul(p->value, NULL, 10); - freeItem(p); - - from = segments[id].from; - to = segments[id].to; - - x0 = nodes[from].lon; - y0 = nodes[from].lat; - x1 = nodes[to].lon; - y1 = nodes[to].lat; - add_segment(x0,y0,x1,y1); - } - return build_geometry(polygon); -} - - -void StartElement(xmlTextReaderPtr reader, const xmlChar *name) -{ - xmlChar *xid, *xlat, *xlon, *xfrom, *xto, *xk, *xv; - unsigned int id, to, from; - double lon, lat; - char *k; - - if (xmlStrEqual(name, BAD_CAST "node")) { - struct osmNode *node; - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); - xlon = xmlTextReaderGetAttribute(reader, BAD_CAST "lon"); - xlat = xmlTextReaderGetAttribute(reader, BAD_CAST "lat"); - assert(xid); assert(xlon); assert(xlat); - id = strtoul((char *)xid, NULL, 10); - lon = strtod((char *)xlon, NULL); - lat = strtod((char *)xlat, NULL); - - assert(id > 0); - assert(id < MAX_ID_NODE); - - if (id > max_node) - max_node = id; - - count_all_node++; - if (count_all_node%10000 == 0) - fprintf(stderr, "\rProcessing: Node(%dk)", count_all_node/1000); - - node = &nodes[id]; - node->lon = lon; - node->lat = lat; - - DEBUG("NODE(%d) %f %f\n", id, lon, lat); - addItem(&keys, "id", (char *)xid, 0); - - xmlFree(xid); - xmlFree(xlon); - xmlFree(xlat); - } else if (xmlStrEqual(name, BAD_CAST "segment")) { - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); - xfrom = xmlTextReaderGetAttribute(reader, BAD_CAST "from"); - xto = xmlTextReaderGetAttribute(reader, BAD_CAST "to"); - assert(xid); assert(xfrom); assert(xto); - id = strtoul((char *)xid, NULL, 10); - from = strtoul((char *)xfrom, NULL, 10); - to = strtoul((char *)xto, NULL, 10); - - assert(id > 0); - assert(id < MAX_ID_SEGMENT); - - if (id > max_segment) - max_segment = id; - - if (count_all_segment == 0) - fprintf(stderr, "\n"); - - count_all_segment++; - if (count_all_segment%10000 == 0) - fprintf(stderr, "\rProcessing: Segment(%dk)", count_all_segment/1000); - - if (!nodes[to].lat && !nodes[to].lon) { - DEBUG("SEGMENT(%d), NODE(%d) is missing\n", id, to); - } else if (!nodes[from].lat && !nodes[from].lon) { - DEBUG("SEGMENT(%d), NODE(%d) is missing\n", id, from); - } else { - if (from != to) { - struct osmSegment *segment; - segment = &segments[id]; - segment->to = to; - segment->from = from; - - count_segment++; - DEBUG("SEGMENT(%d) %d, %d\n", id, from, to); - } - } - - xmlFree(xid); - xmlFree(xfrom); - xmlFree(xto); - } else if (xmlStrEqual(name, BAD_CAST "tag")) { - char *p; - xk = xmlTextReaderGetAttribute(reader, BAD_CAST "k"); - xv = xmlTextReaderGetAttribute(reader, BAD_CAST "v"); - assert(xk); assert(xv); - k = (char *)xmlStrdup(xk); - - while ((p = strchr(k, ':'))) - *p = '_'; - while ((p = strchr(k, ' '))) - *p = '_'; - - addItem(&tags, k, (char *)xv, 0); - DEBUG("\t%s = %s\n", xk, xv); - xmlFree(k); - xmlFree(xk); - xmlFree(xv); - } else if (xmlStrEqual(name, BAD_CAST "way")) { - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); - assert(xid); - id = strtoul((char *)xid, NULL, 10); - addItem(&keys, "id", (char *)xid, 0); - DEBUG("WAY(%s)\n", xid); - - if (id > max_way) - max_way = id; - - if (count_all_way == 0) - fprintf(stderr, "\n"); - - count_all_way++; - if (count_all_way%1000 == 0) - fprintf(stderr, "\rProcessing: Way(%dk)", count_all_way/1000); - - xmlFree(xid); - } else if (xmlStrEqual(name, BAD_CAST "seg")) { - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); - assert(xid); - id = strtoul((char *)xid, NULL, 10); - if (!id || (id > MAX_ID_SEGMENT)) - DEBUG("\tSEG(%s) - invalid segment ID\n", xid); - else if (!segments[id].from || !segments[id].to) - DEBUG("\tSEG(%s) - missing segment\n", xid); - else { - if (addItem(&segs, "id", (char *)xid, 1)) { - const char *way_id = getItem(&keys, "id"); - if (!way_id) way_id = "???"; - //fprintf(stderr, "Way %s with duplicate segment id %d\n", way_id, id); - count_way_seg++; - } - DEBUG("\tSEG(%s)\n", xid); - } - xmlFree(xid); - } else if (xmlStrEqual(name, BAD_CAST "osm")) { - /* ignore */ - } else { - fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); - } -} - -void EndElement(xmlTextReaderPtr reader, const xmlChar *name) -{ - unsigned int id; - - DEBUG("%s: %s\n", __FUNCTION__, name); - - if (xmlStrEqual(name, BAD_CAST "node")) { - int i; - char *values = NULL, *names = NULL; - char *osm_id = getItem(&keys, "id"); - if (!osm_id) { - fprintf(stderr, "%s: Node ID not in keys\n", __FUNCTION__); - resetList(&keys); - resetList(&tags); - return; - } - id = strtoul(osm_id, NULL, 10); - //assert(nodes[id].lat && nodes[id].lon); - for (i=0; i < sizeof(exportTags) / sizeof(exportTags[0]); i++) { - char *v; - if ((v = getItem(&tags, exportTags[i].name))) { - if (values) { - char *oldval = values, *oldnam = names; - asprintf(&names, "%s,\"%s\"", oldnam, exportTags[i].name); - asprintf(&values, "%s,$$%s$$", oldval, v); - free(oldnam); - free(oldval); - } else { - asprintf(&names, "\"%s\"", exportTags[i].name); - asprintf(&values, "$$%s$$", v); - } - } - } - if (values) { - count_node++; - printf("insert into %s (osm_id,%s,way) values " - "(%s,%s,GeomFromText('POINT(%.15g %.15g)',4326));\n", - table_name_point,names,osm_id,values,nodes[id].lon, nodes[id].lat); - } - resetList(&keys); - resetList(&tags); - free(values); - free(names); - } else if (xmlStrEqual(name, BAD_CAST "segment")) { - resetList(&tags); - } else if (xmlStrEqual(name, BAD_CAST "tag")) { - /* Separate tag list so tag stack unused */ - } else if (xmlStrEqual(name, BAD_CAST "way")) { - int i, polygon = 0; - char *values = NULL, *names = NULL; - char *osm_id = getItem(&keys, "id"); - - if (!osm_id) { - fprintf(stderr, "%s: WAY ID not in keys\n", __FUNCTION__); - resetList(&keys); - resetList(&tags); - resetList(&segs); - return; - } - - if (!listHasData(&segs)) { - DEBUG("%s: WAY(%s) has no segments\n", __FUNCTION__, osm_id); - resetList(&keys); - resetList(&tags); - resetList(&segs); - return; - } - id = strtoul(osm_id, NULL, 10); - - for (i=0; i < sizeof(exportTags) / sizeof(exportTags[0]); i++) { - char *v; - if ((v = getItem(&tags, exportTags[i].name))) { - if (values) { - char *oldval = values, *oldnam = names; - asprintf(&names, "%s,\"%s\"", oldnam, exportTags[i].name); - asprintf(&values, "%s,$$%s$$", oldval, v); - free(oldnam); - free(oldval); - } else { - asprintf(&names, "\"%s\"", exportTags[i].name); - asprintf(&values, "$$%s$$", v); - } - polygon |= exportTags[i].polygon; - } - } - if (values) { - size_t wkt_size = WKT(polygon); - - if (wkt_size) - { - unsigned i; - for (i=0;i. +# +# This macro calls: +# +# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# +# And sets: +# +# HAVE_BOOST +# +# LICENSE +# +# Copyright (c) 2008 Thomas Porschberg +# Copyright (c) 2009 Peter Adolphs +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 26 + +AC_DEFUN([AX_BOOST_BASE], +[ +AC_ARG_WITH([boost], + [AS_HELP_STRING([--with-boost@<:@=ARG@:>@], + [use Boost library from a standard location (ARG=yes), + from the specified location (ARG=), + or disable it (ARG=no) + @<:@ARG=yes@:>@ ])], + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ac_boost_path="" + else + want_boost="yes" + ac_boost_path="$withval" + fi + ], + [want_boost="yes"]) + + +AC_ARG_WITH([boost-libdir], + AS_HELP_STRING([--with-boost-libdir=LIB_DIR], + [Force given directory for boost libraries. Note that this will override library path detection, so use this parameter only if default library detection fails and you know exactly where your boost libraries are located.]), + [ + if test -d "$withval" + then + ac_boost_lib_path="$withval" + else + AC_MSG_ERROR(--with-boost-libdir expected directory name) + fi + ], + [ac_boost_lib_path=""] +) + +if test "x$want_boost" = "xyes"; then + boost_lib_version_req=ifelse([$1], ,1.20.0,$1) + boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'` + boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'` + boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'` + boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'` + if test "x$boost_lib_version_req_sub_minor" = "x" ; then + boost_lib_version_req_sub_minor="0" + fi + WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor` + AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req) + succeeded=no + + dnl On 64-bit systems check for system libraries in both lib64 and lib. + dnl The former is specified by FHS, but e.g. Debian does not adhere to + dnl this (as it rises problems for generic multi-arch support). + dnl The last entry in the list is chosen by default when no libraries + dnl are found, e.g. when only header-only libraries are installed! + libsubdirs="lib" + ax_arch=`uname -m` + case $ax_arch in + x86_64) + libsubdirs="lib64 libx32 lib lib64" + ;; + ppc64|s390x|sparc64|aarch64|ppc64le) + libsubdirs="lib64 lib lib64 ppc64le" + ;; + esac + + dnl allow for real multi-arch paths e.g. /usr/lib/x86_64-linux-gnu. Give + dnl them priority over the other paths since, if libs are found there, they + dnl are almost assuredly the ones desired. + AC_REQUIRE([AC_CANONICAL_HOST]) + libsubdirs="lib/${host_cpu}-${host_os} $libsubdirs" + + case ${host_cpu} in + i?86) + libsubdirs="lib/i386-${host_os} $libsubdirs" + ;; + esac + + dnl first we check the system location for boost libraries + dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl or if you install boost with RPM + if test "$ac_boost_path" != ""; then + BOOST_CPPFLAGS="-I$ac_boost_path/include" + for ac_boost_path_tmp in $libsubdirs; do + if test -d "$ac_boost_path"/"$ac_boost_path_tmp" ; then + BOOST_LDFLAGS="-L$ac_boost_path/$ac_boost_path_tmp" + break + fi + done + elif test "$cross_compiling" != yes; then + for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then + for libsubdir in $libsubdirs ; do + if ls "$ac_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + BOOST_LDFLAGS="-L$ac_boost_path_tmp/$libsubdir" + BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include" + break; + fi + done + fi + + dnl overwrite ld flags if we have required special directory with + dnl --with-boost-libdir parameter + if test "$ac_boost_lib_path" != ""; then + BOOST_LDFLAGS="-L$ac_boost_lib_path" + fi + + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_REQUIRE([AC_PROG_CXX]) + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + + + + dnl if we found no boost with system layout we search for boost libraries + dnl built and installed without the --layout=system option or for a staged(not installed) version + if test "x$succeeded" != "xyes"; then + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + BOOST_CPPFLAGS= + BOOST_LDFLAGS= + _version=0 + if test "$ac_boost_path" != ""; then + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + fi + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE" + done + dnl if nothing found search for layout used in Windows distributions + if test -z "$BOOST_CPPFLAGS"; then + if test -d "$ac_boost_path/boost" && test -r "$ac_boost_path/boost"; then + BOOST_CPPFLAGS="-I$ac_boost_path" + fi + fi + fi + else + if test "$cross_compiling" != yes; then + for ac_boost_path in /usr /usr/local /opt /opt/local ; do + if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then + for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do + _version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'` + V_CHECK=`expr $_version_tmp \> $_version` + if test "$V_CHECK" = "1" ; then + _version=$_version_tmp + best_path=$ac_boost_path + fi + done + fi + done + + VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` + BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" + if test "$ac_boost_lib_path" = ""; then + for libsubdir in $libsubdirs ; do + if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + BOOST_LDFLAGS="-L$best_path/$libsubdir" + fi + fi + + if test "x$BOOST_ROOT" != "x"; then + for libsubdir in $libsubdirs ; do + if ls "$BOOST_ROOT/stage/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi + done + if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/$libsubdir" && test -r "$BOOST_ROOT/stage/$libsubdir"; then + version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'` + stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` + stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` + V_CHECK=`expr $stage_version_shorten \>\= $_version` + if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then + AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) + BOOST_CPPFLAGS="-I$BOOST_ROOT" + BOOST_LDFLAGS="-L$BOOST_ROOT/stage/$libsubdir" + fi + fi + fi + fi + + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_LANG_PUSH(C++) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if BOOST_VERSION >= $WANT_BOOST_VERSION + // Everything is okay + #else + # error Boost version is too old + #endif + ]])],[ + AC_MSG_RESULT(yes) + succeeded=yes + found_system=yes + ],[ + ]) + AC_LANG_POP([C++]) + fi + + if test "$succeeded" != "yes" ; then + if test "$_version" = "0" ; then + AC_MSG_NOTICE([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in . See http://randspringer.de/boost for more documentation.]]) + else + AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).]) + fi + # execute ACTION-IF-NOT-FOUND (if present): + ifelse([$3], , :, [$3]) + else + AC_SUBST(BOOST_CPPFLAGS) + AC_SUBST(BOOST_LDFLAGS) + AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available]) + # execute ACTION-IF-FOUND (if present): + ifelse([$2], , :, [$2]) + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" +fi + +]) diff -Nru osm2pgsql-0.82.0/m4/ax_boost_filesystem.m4 osm2pgsql-0.88.1/m4/ax_boost_filesystem.m4 --- osm2pgsql-0.82.0/m4/ax_boost_filesystem.m4 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_boost_filesystem.m4 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,118 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_boost_filesystem.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_FILESYSTEM +# +# DESCRIPTION +# +# Test for Filesystem library from the Boost C++ libraries. The macro +# requires a preceding call to AX_BOOST_BASE. Further documentation is +# available at . +# +# This macro calls: +# +# AC_SUBST(BOOST_FILESYSTEM_LIB) +# +# And sets: +# +# HAVE_BOOST_FILESYSTEM +# +# LICENSE +# +# Copyright (c) 2009 Thomas Porschberg +# Copyright (c) 2009 Michael Tindal +# Copyright (c) 2009 Roman Rybalko +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 26 + +AC_DEFUN([AX_BOOST_FILESYSTEM], +[ + AC_ARG_WITH([boost-filesystem], + AS_HELP_STRING([--with-boost-filesystem@<:@=special-lib@:>@], + [use the Filesystem library from boost - it is possible to specify a certain library for the linker + e.g. --with-boost-filesystem=boost_filesystem-gcc-mt ]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ax_boost_user_filesystem_lib="" + else + want_boost="yes" + ax_boost_user_filesystem_lib="$withval" + fi + ], + [want_boost="yes"] + ) + + if test "x$want_boost" = "xyes"; then + AC_REQUIRE([AC_PROG_CC]) + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + LIBS_SAVED=$LIBS + LIBS="$LIBS $BOOST_SYSTEM_LIB" + export LIBS + + AC_CACHE_CHECK(whether the Boost::Filesystem library is available, + ax_cv_boost_filesystem, + [AC_LANG_PUSH([C++]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], + [[using namespace boost::filesystem; + path my_path( "foo/bar/data.txt" ); + return 0;]])], + ax_cv_boost_filesystem=yes, ax_cv_boost_filesystem=no) + AC_LANG_POP([C++]) + ]) + if test "x$ax_cv_boost_filesystem" = "xyes"; then + AC_DEFINE(HAVE_BOOST_FILESYSTEM,,[define if the Boost::Filesystem library is available]) + BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` + if test "x$ax_boost_user_filesystem_lib" = "x"; then + for libextension in `ls -r $BOOSTLIBDIR/libboost_filesystem* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], + [link_filesystem="no"]) + done + if test "x$link_filesystem" != "xyes"; then + for libextension in `ls -r $BOOSTLIBDIR/boost_filesystem* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], + [link_filesystem="no"]) + done + fi + else + for ax_lib in $ax_boost_user_filesystem_lib boost_filesystem-$ax_boost_user_filesystem_lib; do + AC_CHECK_LIB($ax_lib, exit, + [BOOST_FILESYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_FILESYSTEM_LIB) link_filesystem="yes"; break], + [link_filesystem="no"]) + done + + fi + if test "x$ax_lib" = "x"; then + AC_MSG_ERROR(Could not find a version of the library!) + fi + if test "x$link_filesystem" != "xyes"; then + AC_MSG_ERROR(Could not link against $ax_lib !) + fi + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + LIBS="$LIBS_SAVED" + fi +]) diff -Nru osm2pgsql-0.82.0/m4/ax_boost_system.m4 osm2pgsql-0.88.1/m4/ax_boost_system.m4 --- osm2pgsql-0.82.0/m4/ax_boost_system.m4 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_boost_system.m4 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,120 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_boost_system.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_SYSTEM +# +# DESCRIPTION +# +# Test for System library from the Boost C++ libraries. The macro requires +# a preceding call to AX_BOOST_BASE. Further documentation is available at +# . +# +# This macro calls: +# +# AC_SUBST(BOOST_SYSTEM_LIB) +# +# And sets: +# +# HAVE_BOOST_SYSTEM +# +# LICENSE +# +# Copyright (c) 2008 Thomas Porschberg +# Copyright (c) 2008 Michael Tindal +# Copyright (c) 2008 Daniel Casimiro +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 17 + +AC_DEFUN([AX_BOOST_SYSTEM], +[ + AC_ARG_WITH([boost-system], + AS_HELP_STRING([--with-boost-system@<:@=special-lib@:>@], + [use the System library from boost - it is possible to specify a certain library for the linker + e.g. --with-boost-system=boost_system-gcc-mt ]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ax_boost_user_system_lib="" + else + want_boost="yes" + ax_boost_user_system_lib="$withval" + fi + ], + [want_boost="yes"] + ) + + if test "x$want_boost" = "xyes"; then + AC_REQUIRE([AC_PROG_CC]) + AC_REQUIRE([AC_CANONICAL_BUILD]) + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_CACHE_CHECK(whether the Boost::System library is available, + ax_cv_boost_system, + [AC_LANG_PUSH([C++]) + CXXFLAGS_SAVE=$CXXFLAGS + + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], + [[boost::system::system_category]])], + ax_cv_boost_system=yes, ax_cv_boost_system=no) + CXXFLAGS=$CXXFLAGS_SAVE + AC_LANG_POP([C++]) + ]) + if test "x$ax_cv_boost_system" = "xyes"; then + AC_SUBST(BOOST_CPPFLAGS) + + AC_DEFINE(HAVE_BOOST_SYSTEM,,[define if the Boost::System library is available]) + BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` + + LDFLAGS_SAVE=$LDFLAGS + if test "x$ax_boost_user_system_lib" = "x"; then + for libextension in `ls -r $BOOSTLIBDIR/libboost_system* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], + [link_system="no"]) + done + if test "x$link_system" != "xyes"; then + for libextension in `ls -r $BOOSTLIBDIR/boost_system* 2>/dev/null | sed 's,.*/,,' | sed -e 's,\..*,,'` ; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], + [link_system="no"]) + done + fi + + else + for ax_lib in $ax_boost_user_system_lib boost_system-$ax_boost_user_system_lib; do + AC_CHECK_LIB($ax_lib, exit, + [BOOST_SYSTEM_LIB="-l$ax_lib"; AC_SUBST(BOOST_SYSTEM_LIB) link_system="yes"; break], + [link_system="no"]) + done + + fi + if test "x$ax_lib" = "x"; then + AC_MSG_ERROR(Could not find a version of the library!) + fi + if test "x$link_system" = "xno"; then + AC_MSG_ERROR(Could not link against $ax_lib !) + fi + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + fi +]) diff -Nru osm2pgsql-0.82.0/m4/ax_boost_thread.m4 osm2pgsql-0.88.1/m4/ax_boost_thread.m4 --- osm2pgsql-0.82.0/m4/ax_boost_thread.m4 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_boost_thread.m4 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,149 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_boost_thread.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_BOOST_THREAD +# +# DESCRIPTION +# +# Test for Thread library from the Boost C++ libraries. The macro requires +# a preceding call to AX_BOOST_BASE. Further documentation is available at +# . +# +# This macro calls: +# +# AC_SUBST(BOOST_THREAD_LIB) +# +# And sets: +# +# HAVE_BOOST_THREAD +# +# LICENSE +# +# Copyright (c) 2009 Thomas Porschberg +# Copyright (c) 2009 Michael Tindal +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 27 + +AC_DEFUN([AX_BOOST_THREAD], +[ + AC_ARG_WITH([boost-thread], + AS_HELP_STRING([--with-boost-thread@<:@=special-lib@:>@], + [use the Thread library from boost - it is possible to specify a certain library for the linker + e.g. --with-boost-thread=boost_thread-gcc-mt ]), + [ + if test "$withval" = "no"; then + want_boost="no" + elif test "$withval" = "yes"; then + want_boost="yes" + ax_boost_user_thread_lib="" + else + want_boost="yes" + ax_boost_user_thread_lib="$withval" + fi + ], + [want_boost="yes"] + ) + + if test "x$want_boost" = "xyes"; then + AC_REQUIRE([AC_PROG_CC]) + AC_REQUIRE([AC_CANONICAL_BUILD]) + CPPFLAGS_SAVED="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + export CPPFLAGS + + LDFLAGS_SAVED="$LDFLAGS" + LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" + export LDFLAGS + + AC_CACHE_CHECK(whether the Boost::Thread library is available, + ax_cv_boost_thread, + [AC_LANG_PUSH([C++]) + CXXFLAGS_SAVE=$CXXFLAGS + + if test "x$host_os" = "xsolaris" ; then + CXXFLAGS="-pthreads $CXXFLAGS" + elif test "x$host_os" = "xmingw32" ; then + CXXFLAGS="-mthreads $CXXFLAGS" + else + CXXFLAGS="-pthread $CXXFLAGS" + fi + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include ]], + [[boost::thread_group thrds; + return 0;]])], + ax_cv_boost_thread=yes, ax_cv_boost_thread=no) + CXXFLAGS=$CXXFLAGS_SAVE + AC_LANG_POP([C++]) + ]) + if test "x$ax_cv_boost_thread" = "xyes"; then + if test "x$host_os" = "xsolaris" ; then + BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS" + elif test "x$host_os" = "xmingw32" ; then + BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS" + else + BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS" + fi + + AC_SUBST(BOOST_CPPFLAGS) + + AC_DEFINE(HAVE_BOOST_THREAD,,[define if the Boost::Thread library is available]) + BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'` + + LDFLAGS_SAVE=$LDFLAGS + case "x$host_os" in + *bsd* ) + LDFLAGS="-pthread $LDFLAGS" + break; + ;; + esac + if test "x$ax_boost_user_thread_lib" = "x"; then + for libextension in `ls -r $BOOSTLIBDIR/libboost_thread* 2>/dev/null | sed 's,.*/lib,,' | sed 's,\..*,,'`; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break], + [link_thread="no"]) + done + if test "x$link_thread" != "xyes"; then + for libextension in `ls -r $BOOSTLIBDIR/boost_thread* 2>/dev/null | sed 's,.*/,,' | sed 's,\..*,,'`; do + ax_lib=${libextension} + AC_CHECK_LIB($ax_lib, exit, + [BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break], + [link_thread="no"]) + done + fi + + else + for ax_lib in $ax_boost_user_thread_lib boost_thread-$ax_boost_user_thread_lib; do + AC_CHECK_LIB($ax_lib, exit, + [BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break], + [link_thread="no"]) + done + + fi + if test "x$ax_lib" = "x"; then + AC_MSG_ERROR(Could not find a version of the library!) + fi + if test "x$link_thread" = "xno"; then + AC_MSG_ERROR(Could not link against $ax_lib !) + else + case "x$host_os" in + *bsd* ) + BOOST_LDFLAGS="-pthread $BOOST_LDFLAGS" + break; + ;; + esac + + fi + fi + + CPPFLAGS="$CPPFLAGS_SAVED" + LDFLAGS="$LDFLAGS_SAVED" + fi +]) diff -Nru osm2pgsql-0.82.0/m4/ax_compare_version.m4 osm2pgsql-0.88.1/m4/ax_compare_version.m4 --- osm2pgsql-0.82.0/m4/ax_compare_version.m4 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_compare_version.m4 2015-08-15 05:28:11.000000000 +0000 @@ -175,4 +175,3 @@ m4_ifvaln([$5],[else $5])dnl fi ]) dnl AX_COMPARE_VERSION - diff -Nru osm2pgsql-0.82.0/m4/ax_compile_check_sizeof.m4 osm2pgsql-0.88.1/m4/ax_compile_check_sizeof.m4 --- osm2pgsql-0.82.0/m4/ax_compile_check_sizeof.m4 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_compile_check_sizeof.m4 2015-08-15 05:28:11.000000000 +0000 @@ -111,4 +111,4 @@ AC_DEFINE_UNQUOTED(AC_TYPE_NAME, $AC_CV_NAME, [The number of bytes in type $1]) undefine([AC_TYPE_NAME])dnl undefine([AC_CV_NAME])dnl -]) \ No newline at end of file +]) diff -Nru osm2pgsql-0.82.0/m4/ax_cxx_compile_stdcxx_11.m4 osm2pgsql-0.88.1/m4/ax_cxx_compile_stdcxx_11.m4 --- osm2pgsql-0.82.0/m4/ax_cxx_compile_stdcxx_11.m4 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_cxx_compile_stdcxx_11.m4 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,142 @@ +# ============================================================================ +# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_11.html +# ============================================================================ +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_11([ext|noext],[mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++11 +# standard; if necessary, add switches to CXXFLAGS to enable support. +# +# The first argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The second argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline C++11 support is required and that the macro +# should error out if no mode with that support is found. If specified +# 'optional', then configuration proceeds regardless, after defining +# HAVE_CXX11 if and only if a supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014 Alexey Sokolov +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 4 + +m4_define([_AX_CXX_COMPILE_STDCXX_11_testbody], [[ + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + struct Base { + virtual void f() {} + }; + struct Child : public Base { + virtual void f() override {} + }; + + typedef check> right_angle_brackets; + + int a; + decltype(a) b; + + typedef check check_type; + check_type c; + check_type&& cr = static_cast(c); + + auto d = a; + auto l = [](){}; +]]) + +AC_DEFUN([AX_CXX_COMPILE_STDCXX_11], [dnl + m4_if([$1], [], [], + [$1], [ext], [], + [$1], [noext], [], + [m4_fatal([invalid argument `$1' to AX_CXX_COMPILE_STDCXX_11])])dnl + m4_if([$2], [], [ax_cxx_compile_cxx11_required=true], + [$2], [mandatory], [ax_cxx_compile_cxx11_required=true], + [$2], [optional], [ax_cxx_compile_cxx11_required=false], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX_11])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + AC_CACHE_CHECK(whether $CXX supports C++11 features by default, + ax_cv_cxx_compile_cxx11, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], + [ax_cv_cxx_compile_cxx11=yes], + [ax_cv_cxx_compile_cxx11=no])]) + if test x$ax_cv_cxx_compile_cxx11 = xyes; then + ac_success=yes + fi + + m4_if([$1], [noext], [], [dnl + if test x$ac_success = xno; then + for switch in -std=gnu++11 -std=gnu++0x; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx11_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++11 features with $switch, + $cachevar, + [ac_save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXXFLAGS="$ac_save_CXXFLAGS"]) + if eval test x\$$cachevar = xyes; then + CXXFLAGS="$CXXFLAGS $switch" + ac_success=yes + break + fi + done + fi]) + + m4_if([$1], [ext], [], [dnl + if test x$ac_success = xno; then + for switch in -std=c++11 -std=c++0x; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx11_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++11 features with $switch, + $cachevar, + [ac_save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXXFLAGS="$ac_save_CXXFLAGS"]) + if eval test x\$$cachevar = xyes; then + CXXFLAGS="$CXXFLAGS $switch" + ac_success=yes + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx11_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++11 language features is required.]) + fi + else + if test x$ac_success = xno; then + HAVE_CXX11=0 + AC_MSG_NOTICE([No compiler with C++11 support was found]) + else + HAVE_CXX11=1 + AC_DEFINE(HAVE_CXX11,1, + [define if the compiler supports basic C++11 syntax]) + fi + + AC_SUBST(HAVE_CXX11) + fi +]) diff -Nru osm2pgsql-0.82.0/m4/ax_lib_geos.m4 osm2pgsql-0.88.1/m4/ax_lib_geos.m4 --- osm2pgsql-0.82.0/m4/ax_lib_geos.m4 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_lib_geos.m4 2015-08-15 05:28:11.000000000 +0000 @@ -90,10 +90,10 @@ GEOS_VERSION=`$GEOS_CONFIG --version` dnl Headers are in a different package in Debian, so check again. + ac_save_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $GEOS_CFLAGS" AC_CHECK_HEADER([geos/version.h], [], [AC_MSG_ERROR([development headers for geos not found])]) - echo $ac_save_CPPFLAGS CPPFLAGS="$ac_save_CPPFLAGS" AC_DEFINE([HAVE_GEOS], [1], diff -Nru osm2pgsql-0.82.0/m4/ax_lib_postgresql.m4 osm2pgsql-0.88.1/m4/ax_lib_postgresql.m4 --- osm2pgsql-0.82.0/m4/ax_lib_postgresql.m4 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_lib_postgresql.m4 2015-08-15 05:28:11.000000000 +0000 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.nongnu.org/autoconf-archive/ax_lib_postgresql.html +# http://www.gnu.org/software/autoconf-archive/ax_lib_postgresql.html # =========================================================================== # # SYNOPSIS @@ -27,7 +27,7 @@ # # This macro calls: # -# AC_SUBST(POSTGRESQL_CFLAGS) +# AC_SUBST(POSTGRESQL_CPPFLAGS) # AC_SUBST(POSTGRESQL_LDFLAGS) # AC_SUBST(POSTGRESQL_LIBS) # AC_SUBST(POSTGRESQL_VERSION) @@ -39,15 +39,19 @@ # LICENSE # # Copyright (c) 2008 Mateusz Loskot +# Copyright (c) 2014 Sree Harsha Totakura # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice -# and this notice are preserved. +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 12 AC_DEFUN([AX_LIB_POSTGRESQL], [ AC_ARG_WITH([postgresql], - AC_HELP_STRING([--with-postgresql=@<:@ARG@:>@], + AS_HELP_STRING([--with-postgresql=@<:@ARG@:>@], [use PostgreSQL library @<:@default=yes@:>@, optionally specify path to pg_config] ), [ @@ -63,7 +67,7 @@ [want_postgresql="yes"] ) - POSTGRESQL_CFLAGS="" + POSTGRESQL_CPPFLAGS="" POSTGRESQL_LDFLAGS="" POSTGRESQL_LIBS="" POSTGRESQL_VERSION="" @@ -87,22 +91,16 @@ if test "$PG_CONFIG" != "no"; then AC_MSG_CHECKING([for PostgreSQL libraries]) - POSTGRESQL_CFLAGS="-I`$PG_CONFIG --includedir`" - POSTGRESQL_SERVER_CFLAGS="-I`$PG_CONFIG --includedir-server`" + POSTGRESQL_CPPFLAGS="-I`$PG_CONFIG --includedir`" POSTGRESQL_LDFLAGS="-L`$PG_CONFIG --libdir`" - POSTGRESQL_LIBS="-lpq" - + POSTGRESQL_LIBS="-lpq" POSTGRESQL_VERSION=`$PG_CONFIG --version | sed -e 's#PostgreSQL ##'` AC_DEFINE([HAVE_POSTGRESQL], [1], [Define to 1 if PostgreSQL libraries are available]) - POSTGRESQL_PGXS=`$PG_CONFIG --pgxs` - if test -f "$POSTGRESQL_PGXS" - then - found_postgresql="yes" - AC_MSG_RESULT([yes]) - fi + found_postgresql="yes" + AC_MSG_RESULT([yes]) else found_postgresql="no" AC_MSG_RESULT([no]) @@ -151,14 +149,16 @@ AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) + AC_DEFINE([HAVE_POSTGRESQL], [0], + [A required version of PostgreSQL is not found]) + POSTGRESQL_CPPFLAGS="" + POSTGRESQL_LDFLAGS="" + POSTGRESQL_LIBS="" fi fi - AC_SUBST([POSTGRESQL_PGXS]) AC_SUBST([POSTGRESQL_VERSION]) - AC_SUBST([POSTGRESQL_CFLAGS]) - AC_SUBST([POSTGRESQL_SERVER_CFLAGS]) + AC_SUBST([POSTGRESQL_CPPFLAGS]) AC_SUBST([POSTGRESQL_LDFLAGS]) AC_SUBST([POSTGRESQL_LIBS]) ]) - diff -Nru osm2pgsql-0.82.0/m4/ax_lib_protobuf_c.m4 osm2pgsql-0.88.1/m4/ax_lib_protobuf_c.m4 --- osm2pgsql-0.82.0/m4/ax_lib_protobuf_c.m4 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_lib_protobuf_c.m4 2015-08-15 05:28:11.000000000 +0000 @@ -207,7 +207,9 @@ CFLAGS="$CFLAGS $PROTOBUF_C_CFLAGS" AX_COMPARE_VERSION([$protobuf_c_wanted_version], [ge], [0.14], [AC_CHECK_MEMBER([ProtobufCFieldDescriptor.packed],, - [protobuf_c_version_ok="no"], + [AC_CHECK_MEMBER([ProtobufCFieldDescriptor.flags],, + [protobuf_c_version_ok="no"], + [[#include ]])], [[#include ] ]) ]) diff -Nru osm2pgsql-0.82.0/m4/ax_lua.m4 osm2pgsql-0.88.1/m4/ax_lua.m4 --- osm2pgsql-0.82.0/m4/ax_lua.m4 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_lua.m4 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,664 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_lua.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_PROG_LUA[([MINIMUM-VERSION], [TOO-BIG-VERSION], [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])] +# AX_LUA_HEADERS[([ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])] +# AX_LUA_LIBS[([ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])] +# AX_LUA_READLINE[([ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND])] +# +# DESCRIPTION +# +# Detect a Lua interpreter, optionally specifying a minimum and maximum +# version number. Set up important Lua paths, such as the directories in +# which to install scripts and modules (shared libraries). +# +# Also detect Lua headers and libraries. The Lua version contained in the +# header is checked to match the Lua interpreter version exactly. When +# searching for Lua libraries, the version number is used as a suffix. +# This is done with the goal of supporting multiple Lua installs (5.1 and +# 5.2 side-by-side). +# +# A note on compatibility with previous versions: This file has been +# mostly rewritten for serial 18. Most developers should be able to use +# these macros without needing to modify configure.ac. Care has been taken +# to preserve each macro's behavior, but there are some differences: +# +# 1) AX_WITH_LUA is deprecated; it now expands to the exact same thing as +# AX_PROG_LUA with no arguments. +# +# 2) AX_LUA_HEADERS now checks that the version number defined in lua.h +# matches the interpreter version. AX_LUA_HEADERS_VERSION is therefore +# unnecessary, so it is deprecated and does not expand to anything. +# +# 3) The configure flag --with-lua-suffix no longer exists; the user +# should instead specify the LUA precious variable on the command line. +# See the AX_PROG_LUA description for details. +# +# Please read the macro descriptions below for more information. +# +# This file was inspired by Andrew Dalke's and James Henstridge's +# python.m4 and Tom Payne's, Matthieu Moy's, and Reuben Thomas's ax_lua.m4 +# (serial 17). Basically, this file is a mash-up of those two files. I +# like to think it combines the best of the two! +# +# AX_PROG_LUA: Search for the Lua interpreter, and set up important Lua +# paths. Adds precious variable LUA, which may contain the path of the Lua +# interpreter. If LUA is blank, the user's path is searched for an +# suitable interpreter. +# +# If MINIMUM-VERSION is supplied, then only Lua interpreters with a +# version number greater or equal to MINIMUM-VERSION will be accepted. If +# TOO-BIG-VERSION is also supplied, then only Lua interpreters with a +# version number greater or equal to MINIMUM-VERSION and less than +# TOO-BIG-VERSION will be accepted. +# +# The Lua version number, LUA_VERSION, is found from the interpreter, and +# substituted. LUA_PLATFORM is also found, but not currently supported (no +# standard representation). +# +# Finally, the macro finds four paths: +# +# luadir Directory to install Lua scripts. +# pkgluadir $luadir/$PACKAGE +# luaexecdir Directory to install Lua modules. +# pkgluaexecdir $luaexecdir/$PACKAGE +# +# These paths are found based on $prefix, $exec_prefix, Lua's +# package.path, and package.cpath. The first path of package.path +# beginning with $prefix is selected as luadir. The first path of +# package.cpath beginning with $exec_prefix is used as luaexecdir. This +# should work on all reasonable Lua installations. If a path cannot be +# determined, a default path is used. Of course, the user can override +# these later when invoking make. +# +# luadir Default: $prefix/share/lua/$LUA_VERSION +# luaexecdir Default: $exec_prefix/lib/lua/$LUA_VERSION +# +# These directories can be used by Automake as install destinations. The +# variable name minus 'dir' needs to be used as a prefix to the +# appropriate Automake primary, e.g. lua_SCRIPS or luaexec_LIBRARIES. +# +# If an acceptable Lua interpreter is found, then ACTION-IF-FOUND is +# performed, otherwise ACTION-IF-NOT-FOUND is preformed. If ACTION-IF-NOT- +# FOUND is blank, then it will default to printing an error. To prevent +# the default behavior, give ':' as an action. +# +# AX_LUA_HEADERS: Search for Lua headers. Requires that AX_PROG_LUA be +# expanded before this macro. Adds precious variable LUA_INCLUDE, which +# may contain Lua specific include flags, e.g. -I/usr/include/lua5.1. If +# LUA_INCLUDE is blank, then this macro will attempt to find suitable +# flags. +# +# LUA_INCLUDE can be used by Automake to compile Lua modules or +# executables with embedded interpreters. The *_CPPFLAGS variables should +# be used for this purpose, e.g. myprog_CPPFLAGS = $(LUA_INCLUDE). +# +# This macro searches for the header lua.h (and others). The search is +# performed with a combination of CPPFLAGS, CPATH, etc, and LUA_INCLUDE. +# If the search is unsuccessful, then some common directories are tried. +# If the headers are then found, then LUA_INCLUDE is set accordingly. +# +# The paths automatically searched are: +# +# * /usr/include/luaX.Y +# * /usr/include/lua/X.Y +# * /usr/include/luaXY +# * /usr/local/include/luaX.Y +# * /usr/local/include/lua-X.Y +# * /usr/local/include/lua/X.Y +# * /usr/local/include/luaXY +# +# (Where X.Y is the Lua version number, e.g. 5.1.) +# +# The Lua version number found in the headers is always checked to match +# the Lua interpreter's version number. Lua headers with mismatched +# version numbers are not accepted. +# +# If headers are found, then ACTION-IF-FOUND is performed, otherwise +# ACTION-IF-NOT-FOUND is performed. If ACTION-IF-NOT-FOUND is blank, then +# it will default to printing an error. To prevent the default behavior, +# set the action to ':'. +# +# AX_LUA_LIBS: Search for Lua libraries. Requires that AX_PROG_LUA be +# expanded before this macro. Adds precious variable LUA_LIB, which may +# contain Lua specific linker flags, e.g. -llua5.1. If LUA_LIB is blank, +# then this macro will attempt to find suitable flags. +# +# LUA_LIB can be used by Automake to link Lua modules or executables with +# embedded interpreters. The *_LIBADD and *_LDADD variables should be used +# for this purpose, e.g. mymod_LIBADD = $(LUA_LIB). +# +# This macro searches for the Lua library. More technically, it searches +# for a library containing the function lua_load. The search is performed +# with a combination of LIBS, LIBRARY_PATH, and LUA_LIB. +# +# If the search determines that some linker flags are missing, then those +# flags will be added to LUA_LIB. +# +# If libraries are found, then ACTION-IF-FOUND is performed, otherwise +# ACTION-IF-NOT-FOUND is performed. If ACTION-IF-NOT-FOUND is blank, then +# it will default to printing an error. To prevent the default behavior, +# set the action to ':'. +# +# AX_LUA_READLINE: Search for readline headers and libraries. Requires the +# AX_LIB_READLINE macro, which is provided by ax_lib_readline.m4 from the +# Autoconf Archive. +# +# If a readline compatible library is found, then ACTION-IF-FOUND is +# performed, otherwise ACTION-IF-NOT-FOUND is performed. +# +# LICENSE +# +# Copyright (c) 2015 Reuben Thomas +# Copyright (c) 2014 Tim Perkins +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 38 + +dnl ========================================================================= +dnl AX_PROG_LUA([MINIMUM-VERSION], [TOO-BIG-VERSION], +dnl [ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ========================================================================= +AC_DEFUN([AX_PROG_LUA], +[ + dnl Check for required tools. + AC_REQUIRE([AC_PROG_GREP]) + AC_REQUIRE([AC_PROG_SED]) + + dnl Make LUA a precious variable. + AC_ARG_VAR([LUA], [The Lua interpreter, e.g. /usr/bin/lua5.1]) + + dnl Find a Lua interpreter. + m4_define_default([_AX_LUA_INTERPRETER_LIST], + [lua lua5.2 lua52 lua5.1 lua51 lua50]) + + m4_if([$1], [], + [ dnl No version check is needed. Find any Lua interpreter. + AS_IF([test "x$LUA" = 'x'], + [AC_PATH_PROGS([LUA], [_AX_LUA_INTERPRETER_LIST], [:])]) + ax_display_LUA='lua' + + AS_IF([test "x$LUA" != 'x:'], + [ dnl At least check if this is a Lua interpreter. + AC_MSG_CHECKING([if $LUA is a Lua interpreter]) + _AX_LUA_CHK_IS_INTRP([$LUA], + [AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]) + AC_MSG_ERROR([not a Lua interpreter]) + ]) + ]) + ], + [ dnl A version check is needed. + AS_IF([test "x$LUA" != 'x'], + [ dnl Check if this is a Lua interpreter. + AC_MSG_CHECKING([if $LUA is a Lua interpreter]) + _AX_LUA_CHK_IS_INTRP([$LUA], + [AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]) + AC_MSG_ERROR([not a Lua interpreter]) + ]) + dnl Check the version. + m4_if([$2], [], + [_ax_check_text="whether $LUA version >= $1"], + [_ax_check_text="whether $LUA version >= $1, < $2"]) + AC_MSG_CHECKING([$_ax_check_text]) + _AX_LUA_CHK_VER([$LUA], [$1], [$2], + [AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]) + AC_MSG_ERROR([version is out of range for specified LUA])]) + ax_display_LUA=$LUA + ], + [ dnl Try each interpreter until we find one that satisfies VERSION. + m4_if([$2], [], + [_ax_check_text="for a Lua interpreter with version >= $1"], + [_ax_check_text="for a Lua interpreter with version >= $1, < $2"]) + AC_CACHE_CHECK([$_ax_check_text], + [ax_cv_pathless_LUA], + [ for ax_cv_pathless_LUA in _AX_LUA_INTERPRETER_LIST none; do + test "x$ax_cv_pathless_LUA" = 'xnone' && break + _AX_LUA_CHK_IS_INTRP([$ax_cv_pathless_LUA], [], [continue]) + _AX_LUA_CHK_VER([$ax_cv_pathless_LUA], [$1], [$2], [break]) + done + ]) + dnl Set $LUA to the absolute path of $ax_cv_pathless_LUA. + AS_IF([test "x$ax_cv_pathless_LUA" = 'xnone'], + [LUA=':'], + [AC_PATH_PROG([LUA], [$ax_cv_pathless_LUA])]) + ax_display_LUA=$ax_cv_pathless_LUA + ]) + ]) + + AS_IF([test "x$LUA" = 'x:'], + [ dnl Run any user-specified action, or abort. + m4_default([$4], [AC_MSG_ERROR([cannot find suitable Lua interpreter])]) + ], + [ dnl Query Lua for its version number. + AC_CACHE_CHECK([for $ax_display_LUA version], + [ax_cv_lua_version], + [ dnl Get the interpreter version in X.Y format. This should work for + dnl interpreters version 5.0 and beyond. + ax_cv_lua_version=[`$LUA -e ' + -- return a version number in X.Y format + local _, _, ver = string.find(_VERSION, "^Lua (%d+%.%d+)") + print(ver)'`] + ]) + AS_IF([test "x$ax_cv_lua_version" = 'x'], + [AC_MSG_ERROR([invalid Lua version number])]) + AC_SUBST([LUA_VERSION], [$ax_cv_lua_version]) + AC_SUBST([LUA_SHORT_VERSION], [`echo "$LUA_VERSION" | $SED 's|\.||'`]) + + dnl The following check is not supported: + dnl At times (like when building shared libraries) you may want to know + dnl which OS platform Lua thinks this is. + AC_CACHE_CHECK([for $ax_display_LUA platform], + [ax_cv_lua_platform], + [ax_cv_lua_platform=[`$LUA -e 'print("unknown")'`]]) + AC_SUBST([LUA_PLATFORM], [$ax_cv_lua_platform]) + + dnl Use the values of $prefix and $exec_prefix for the corresponding + dnl values of LUA_PREFIX and LUA_EXEC_PREFIX. These are made distinct + dnl variables so they can be overridden if need be. However, the general + dnl consensus is that you shouldn't need this ability. + AC_SUBST([LUA_PREFIX], ['${prefix}']) + AC_SUBST([LUA_EXEC_PREFIX], ['${exec_prefix}']) + + dnl Lua provides no way to query the script directory, and instead + dnl provides LUA_PATH. However, we should be able to make a safe educated + dnl guess. If the built-in search path contains a directory which is + dnl prefixed by $prefix, then we can store scripts there. The first + dnl matching path will be used. + AC_CACHE_CHECK([for $ax_display_LUA script directory], + [ax_cv_lua_luadir], + [ AS_IF([test "x$prefix" = 'xNONE'], + [ax_lua_prefix=$ac_default_prefix], + [ax_lua_prefix=$prefix]) + + dnl Initialize to the default path. + ax_cv_lua_luadir="$LUA_PREFIX/share/lua/$LUA_VERSION" + + dnl Try to find a path with the prefix. + _AX_LUA_FND_PRFX_PTH([$LUA], [$ax_lua_prefix], [script]) + AS_IF([test "x$ax_lua_prefixed_path" != 'x'], + [ dnl Fix the prefix. + _ax_strip_prefix=`echo "$ax_lua_prefix" | $SED 's|.|.|g'` + ax_cv_lua_luadir=`echo "$ax_lua_prefixed_path" | \ + $SED "s|^$_ax_strip_prefix|$LUA_PREFIX|"` + ]) + ]) + AC_SUBST([luadir], [$ax_cv_lua_luadir]) + AC_SUBST([pkgluadir], [\${luadir}/$PACKAGE]) + + dnl Lua provides no way to query the module directory, and instead + dnl provides LUA_PATH. However, we should be able to make a safe educated + dnl guess. If the built-in search path contains a directory which is + dnl prefixed by $exec_prefix, then we can store modules there. The first + dnl matching path will be used. + AC_CACHE_CHECK([for $ax_display_LUA module directory], + [ax_cv_lua_luaexecdir], + [ AS_IF([test "x$exec_prefix" = 'xNONE'], + [ax_lua_exec_prefix=$ax_lua_prefix], + [ax_lua_exec_prefix=$exec_prefix]) + + dnl Initialize to the default path. + ax_cv_lua_luaexecdir="$LUA_EXEC_PREFIX/lib/lua/$LUA_VERSION" + + dnl Try to find a path with the prefix. + _AX_LUA_FND_PRFX_PTH([$LUA], + [$ax_lua_exec_prefix], [module]) + AS_IF([test "x$ax_lua_prefixed_path" != 'x'], + [ dnl Fix the prefix. + _ax_strip_prefix=`echo "$ax_lua_exec_prefix" | $SED 's|.|.|g'` + ax_cv_lua_luaexecdir=`echo "$ax_lua_prefixed_path" | \ + $SED "s|^$_ax_strip_prefix|$LUA_EXEC_PREFIX|"` + ]) + ]) + AC_SUBST([luaexecdir], [$ax_cv_lua_luaexecdir]) + AC_SUBST([pkgluaexecdir], [\${luaexecdir}/$PACKAGE]) + + dnl Run any user specified action. + $3 + ]) +]) + +dnl AX_WITH_LUA is now the same thing as AX_PROG_LUA. +AC_DEFUN([AX_WITH_LUA], +[ + AC_MSG_WARN([[$0 is deprecated, please use AX_PROG_LUA instead]]) + AX_PROG_LUA +]) + + +dnl ========================================================================= +dnl _AX_LUA_CHK_IS_INTRP(PROG, [ACTION-IF-TRUE], [ACTION-IF-FALSE]) +dnl ========================================================================= +AC_DEFUN([_AX_LUA_CHK_IS_INTRP], +[ + dnl A minimal Lua factorial to prove this is an interpreter. This should work + dnl for Lua interpreters version 5.0 and beyond. + _ax_lua_factorial=[`$1 2>/dev/null -e ' + -- a simple factorial + function fact (n) + if n == 0 then + return 1 + else + return n * fact(n-1) + end + end + print("fact(5) is " .. fact(5))'`] + AS_IF([test "$_ax_lua_factorial" = 'fact(5) is 120'], + [$2], [$3]) +]) + + +dnl ========================================================================= +dnl _AX_LUA_CHK_VER(PROG, MINIMUM-VERSION, [TOO-BIG-VERSION], +dnl [ACTION-IF-TRUE], [ACTION-IF-FALSE]) +dnl ========================================================================= +AC_DEFUN([_AX_LUA_CHK_VER], +[ + dnl Check that the Lua version is within the bounds. Only the major and minor + dnl version numbers are considered. This should work for Lua interpreters + dnl version 5.0 and beyond. + _ax_lua_good_version=[`$1 -e ' + -- a script to compare versions + function verstr2num(verstr) + local _, _, majorver, minorver = string.find(verstr, "^(%d+)%.(%d+)") + if majorver and minorver then + return tonumber(majorver) * 100 + tonumber(minorver) + end + end + local minver = verstr2num("$2") + local _, _, trimver = string.find(_VERSION, "^Lua (.*)") + local ver = verstr2num(trimver) + local maxver = verstr2num("$3") or 1e9 + if minver <= ver and ver < maxver then + print("yes") + else + print("no") + end'`] + AS_IF([test "x$_ax_lua_good_version" = "xyes"], + [$4], [$5]) +]) + + +dnl ========================================================================= +dnl _AX_LUA_FND_PRFX_PTH(PROG, PREFIX, SCRIPT-OR-MODULE-DIR) +dnl ========================================================================= +AC_DEFUN([_AX_LUA_FND_PRFX_PTH], +[ + dnl Get the script or module directory by querying the Lua interpreter, + dnl filtering on the given prefix, and selecting the shallowest path. If no + dnl path is found matching the prefix, the result will be an empty string. + dnl The third argument determines the type of search, it can be 'script' or + dnl 'module'. Supplying 'script' will perform the search with package.path + dnl and LUA_PATH, and supplying 'module' will search with package.cpath and + dnl LUA_CPATH. This is done for compatibility with Lua 5.0. + + ax_lua_prefixed_path=[`$1 -e ' + -- get the path based on search type + local searchtype = "$3" + local paths = "" + if searchtype == "script" then + paths = (package and package.path) or LUA_PATH + elseif searchtype == "module" then + paths = (package and package.cpath) or LUA_CPATH + end + -- search for the prefix + local prefix = "'$2'" + local minpath = "" + local mindepth = 1e9 + string.gsub(paths, "(@<:@^;@:>@+)", + function (path) + path = string.gsub(path, "%?.*$", "") + path = string.gsub(path, "/@<:@^/@:>@*$", "") + if string.find(path, prefix) then + local depth = string.len(string.gsub(path, "@<:@^/@:>@", "")) + if depth < mindepth then + minpath = path + mindepth = depth + end + end + end) + print(minpath)'`] +]) + + +dnl ========================================================================= +dnl AX_LUA_HEADERS([ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ========================================================================= +AC_DEFUN([AX_LUA_HEADERS], +[ + dnl Check for LUA_VERSION. + AC_MSG_CHECKING([if LUA_VERSION is defined]) + AS_IF([test "x$LUA_VERSION" != 'x'], + [AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]) + AC_MSG_ERROR([cannot check Lua headers without knowing LUA_VERSION]) + ]) + + dnl Make LUA_INCLUDE a precious variable. + AC_ARG_VAR([LUA_INCLUDE], [The Lua includes, e.g. -I/usr/include/lua5.1]) + + dnl Some default directories to search. + LUA_SHORT_VERSION=`echo "$LUA_VERSION" | $SED 's|\.||'` + m4_define_default([_AX_LUA_INCLUDE_LIST], + [ /usr/include/lua$LUA_VERSION \ + /usr/include/lua-$LUA_VERSION \ + /usr/include/lua/$LUA_VERSION \ + /usr/include/lua$LUA_SHORT_VERSION \ + /usr/local/include/lua$LUA_VERSION \ + /usr/local/include/lua-$LUA_VERSION \ + /usr/local/include/lua/$LUA_VERSION \ + /usr/local/include/lua$LUA_SHORT_VERSION \ + ]) + + dnl Try to find the headers. + _ax_lua_saved_cppflags=$CPPFLAGS + CPPFLAGS="$CPPFLAGS $LUA_INCLUDE" + AC_CHECK_HEADERS([lua.h lualib.h lauxlib.h luaconf.h]) + CPPFLAGS=$_ax_lua_saved_cppflags + + dnl Try some other directories if LUA_INCLUDE was not set. + AS_IF([test "x$LUA_INCLUDE" = 'x' && + test "x$ac_cv_header_lua_h" != 'xyes'], + [ dnl Try some common include paths. + for _ax_include_path in _AX_LUA_INCLUDE_LIST; do + test ! -d "$_ax_include_path" && continue + + AC_MSG_CHECKING([for Lua headers in]) + AC_MSG_RESULT([$_ax_include_path]) + + AS_UNSET([ac_cv_header_lua_h]) + AS_UNSET([ac_cv_header_lualib_h]) + AS_UNSET([ac_cv_header_lauxlib_h]) + AS_UNSET([ac_cv_header_luaconf_h]) + + _ax_lua_saved_cppflags=$CPPFLAGS + CPPFLAGS="$CPPFLAGS -I$_ax_include_path" + AC_CHECK_HEADERS([lua.h lualib.h lauxlib.h luaconf.h]) + CPPFLAGS=$_ax_lua_saved_cppflags + + AS_IF([test "x$ac_cv_header_lua_h" = 'xyes'], + [ LUA_INCLUDE="-I$_ax_include_path" + break + ]) + done + ]) + + AS_IF([test "x$ac_cv_header_lua_h" = 'xyes'], + [ dnl Make a program to print LUA_VERSION defined in the header. + dnl TODO It would be really nice if we could do this without compiling a + dnl program, then it would work when cross compiling. But I'm not sure how + dnl to do this reliably. For now, assume versions match when cross compiling. + + AS_IF([test "x$cross_compiling" != 'xyes'], + [ AC_CACHE_CHECK([for Lua header version], + [ax_cv_lua_header_version], + [ _ax_lua_saved_cppflags=$CPPFLAGS + CPPFLAGS="$CPPFLAGS $LUA_INCLUDE" + AC_RUN_IFELSE( + [ AC_LANG_SOURCE([[ +#include +#include +#include +int main(int argc, char ** argv) +{ + if(argc > 1) printf("%s", LUA_VERSION); + exit(EXIT_SUCCESS); +} +]]) + ], + [ ax_cv_lua_header_version=`./conftest$EXEEXT p | \ + $SED -n "s|^Lua \(@<:@0-9@:>@\{1,\}\.@<:@0-9@:>@\{1,\}\).\{0,\}|\1|p"` + ], + [ax_cv_lua_header_version='unknown']) + CPPFLAGS=$_ax_lua_saved_cppflags + ]) + + dnl Compare this to the previously found LUA_VERSION. + AC_MSG_CHECKING([if Lua header version matches $LUA_VERSION]) + AS_IF([test "x$ax_cv_lua_header_version" = "x$LUA_VERSION"], + [ AC_MSG_RESULT([yes]) + ax_header_version_match='yes' + ], + [ AC_MSG_RESULT([no]) + ax_header_version_match='no' + ]) + ], + [ AC_MSG_WARN([cross compiling so assuming header version number matches]) + ax_header_version_match='yes' + ]) + ]) + + dnl Was LUA_INCLUDE specified? + AS_IF([test "x$ax_header_version_match" != 'xyes' && + test "x$LUA_INCLUDE" != 'x'], + [AC_MSG_ERROR([cannot find headers for specified LUA_INCLUDE])]) + + dnl Test the final result and run user code. + AS_IF([test "x$ax_header_version_match" = 'xyes'], [$1], + [m4_default([$2], [AC_MSG_ERROR([cannot find Lua includes])])]) +]) + +dnl AX_LUA_HEADERS_VERSION no longer exists, use AX_LUA_HEADERS. +AC_DEFUN([AX_LUA_HEADERS_VERSION], +[ + AC_MSG_WARN([[$0 is deprecated, please use AX_LUA_HEADERS instead]]) +]) + + +dnl ========================================================================= +dnl AX_LUA_LIBS([ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ========================================================================= +AC_DEFUN([AX_LUA_LIBS], +[ + dnl TODO Should this macro also check various -L flags? + + dnl Check for LUA_VERSION. + AC_MSG_CHECKING([if LUA_VERSION is defined]) + AS_IF([test "x$LUA_VERSION" != 'x'], + [AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]) + AC_MSG_ERROR([cannot check Lua libs without knowing LUA_VERSION]) + ]) + + dnl Make LUA_LIB a precious variable. + AC_ARG_VAR([LUA_LIB], [The Lua library, e.g. -llua5.1]) + + AS_IF([test "x$LUA_LIB" != 'x'], + [ dnl Check that LUA_LIBS works. + _ax_lua_saved_libs=$LIBS + LIBS="$LIBS $LUA_LIB" + AC_SEARCH_LIBS([lua_load], [], + [_ax_found_lua_libs='yes'], + [_ax_found_lua_libs='no']) + LIBS=$_ax_lua_saved_libs + + dnl Check the result. + AS_IF([test "x$_ax_found_lua_libs" != 'xyes'], + [AC_MSG_ERROR([cannot find libs for specified LUA_LIB])]) + ], + [ dnl First search for extra libs. + _ax_lua_extra_libs='' + + _ax_lua_saved_libs=$LIBS + LIBS="$LIBS $LUA_LIB" + AC_SEARCH_LIBS([exp], [m]) + AC_SEARCH_LIBS([dlopen], [dl]) + LIBS=$_ax_lua_saved_libs + + AS_IF([test "x$ac_cv_search_exp" != 'xno' && + test "x$ac_cv_search_exp" != 'xnone required'], + [_ax_lua_extra_libs="$_ax_lua_extra_libs $ac_cv_search_exp"]) + + AS_IF([test "x$ac_cv_search_dlopen" != 'xno' && + test "x$ac_cv_search_dlopen" != 'xnone required'], + [_ax_lua_extra_libs="$_ax_lua_extra_libs $ac_cv_search_dlopen"]) + + dnl Try to find the Lua libs. + _ax_lua_saved_libs=$LIBS + LIBS="$LIBS $LUA_LIB" + AC_SEARCH_LIBS([lua_load], + [ lua$LUA_VERSION \ + lua$LUA_SHORT_VERSION \ + lua-$LUA_VERSION \ + lua-$LUA_SHORT_VERSION \ + lua \ + ], + [_ax_found_lua_libs='yes'], + [_ax_found_lua_libs='no'], + [$_ax_lua_extra_libs]) + LIBS=$_ax_lua_saved_libs + + AS_IF([test "x$ac_cv_search_lua_load" != 'xno' && + test "x$ac_cv_search_lua_load" != 'xnone required'], + [LUA_LIB="$ac_cv_search_lua_load $_ax_lua_extra_libs"]) + ]) + + dnl Test the result and run user code. + AS_IF([test "x$_ax_found_lua_libs" = 'xyes'], [$1], + [m4_default([$2], [AC_MSG_ERROR([cannot find Lua libs])])]) +]) + + +dnl ========================================================================= +dnl AX_LUA_READLINE([ACTION-IF-FOUND], [ACTION-IF-NOT-FOUND]) +dnl ========================================================================= +AC_DEFUN([AX_LUA_READLINE], +[ + AX_LIB_READLINE + AS_IF([test "x$ac_cv_header_readline_readline_h" != 'x' && + test "x$ac_cv_header_readline_history_h" != 'x'], + [ LUA_LIBS_CFLAGS="-DLUA_USE_READLINE $LUA_LIBS_CFLAGS" + $1 + ], + [$2]) +]) diff -Nru osm2pgsql-0.82.0/m4/ax_pthread.m4 osm2pgsql-0.88.1/m4/ax_pthread.m4 --- osm2pgsql-0.82.0/m4/ax_pthread.m4 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/m4/ax_pthread.m4 2015-08-15 05:28:11.000000000 +0000 @@ -33,6 +33,10 @@ # has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to that name # (e.g. PTHREAD_CREATE_UNDETACHED on AIX). # +# Also HAVE_PTHREAD_PRIO_INHERIT is defined if pthread is found and the +# PTHREAD_PRIO_INHERIT symbol is defined when compiling with +# PTHREAD_CFLAGS. +# # ACTION-IF-FOUND is a list of shell commands to run if a threads library # is found, and ACTION-IF-NOT-FOUND is a list of commands to run it if it # is not found. If ACTION-IF-FOUND is not specified, the default action @@ -45,9 +49,12 @@ # Alejandro Forero Cuervo to the autoconf macro repository. We are also # grateful for the helpful feedback of numerous users. # +# Updated for Autoconf 2.68 by Daniel Richard G. +# # LICENSE # # Copyright (c) 2008 Steven G. Johnson +# Copyright (c) 2011 Daniel Richard G. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -75,13 +82,12 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 11 +#serial 21 AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD]) AC_DEFUN([AX_PTHREAD], [ AC_REQUIRE([AC_CANONICAL_HOST]) -AC_LANG_SAVE -AC_LANG_C +AC_LANG_PUSH([C]) ax_pthread_ok=no # We used to check for pthread.h first, but this fails if pthread.h @@ -97,8 +103,8 @@ save_LIBS="$LIBS" LIBS="$PTHREAD_LIBS $LIBS" AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS]) - AC_TRY_LINK_FUNC(pthread_join, ax_pthread_ok=yes) - AC_MSG_RESULT($ax_pthread_ok) + AC_TRY_LINK_FUNC([pthread_join], [ax_pthread_ok=yes]) + AC_MSG_RESULT([$ax_pthread_ok]) if test x"$ax_pthread_ok" = xno; then PTHREAD_LIBS="" PTHREAD_CFLAGS="" @@ -139,8 +145,8 @@ # --thread-safe: KAI C++ # pthread-config: use pthread-config program (for GNU Pth library) -case "${host_cpu}-${host_os}" in - *solaris*) +case ${host_os} in + solaris*) # On Solaris (at least, for some versions), libc contains stubbed # (non-functional) versions of the pthreads routines, so link-based @@ -153,11 +159,25 @@ ax_pthread_flags="-pthreads pthread -mt -pthread $ax_pthread_flags" ;; - *-darwin*) - ax_pthread_flags="-pthread $ax_pthread_flags" - ;; + darwin*) + ax_pthread_flags="-pthread $ax_pthread_flags" + ;; esac +# Clang doesn't consider unrecognized options an error unless we specify +# -Werror. We throw in some extra Clang-specific options to ensure that +# this doesn't happen for GCC, which also accepts -Werror. + +AC_MSG_CHECKING([if compiler needs -Werror to reject unknown flags]) +save_CFLAGS="$CFLAGS" +ax_pthread_extra_flags="-Werror" +CFLAGS="$CFLAGS $ax_pthread_extra_flags -Wunknown-warning-option -Wsizeof-array-argument" +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([int foo(void);],[foo()])], + [AC_MSG_RESULT([yes])], + [ax_pthread_extra_flags= + AC_MSG_RESULT([no])]) +CFLAGS="$save_CFLAGS" + if test x"$ax_pthread_ok" = xno; then for flag in $ax_pthread_flags; do @@ -171,12 +191,12 @@ PTHREAD_CFLAGS="$flag" ;; - pthread-config) - AC_CHECK_PROG(ax_pthread_config, pthread-config, yes, no) - if test x"$ax_pthread_config" = xno; then continue; fi - PTHREAD_CFLAGS="`pthread-config --cflags`" - PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" - ;; + pthread-config) + AC_CHECK_PROG([ax_pthread_config], [pthread-config], [yes], [no]) + if test x"$ax_pthread_config" = xno; then continue; fi + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; *) AC_MSG_CHECKING([for the pthreads library -l$flag]) @@ -187,7 +207,7 @@ save_LIBS="$LIBS" save_CFLAGS="$CFLAGS" LIBS="$PTHREAD_LIBS $LIBS" - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS $ax_pthread_extra_flags" # Check for various functions. We must include pthread.h, # since some functions may be macros. (On the Sequent, we @@ -198,21 +218,22 @@ # pthread_cleanup_push because it is one of the few pthread # functions on Solaris that doesn't have a non-functional libc stub. # We try pthread_create on general principles. - AC_TRY_LINK([#include - static void routine(void* a) {a=0;} - static void* start_routine(void* a) {return a;}], - [pthread_t th; pthread_attr_t attr; - pthread_create(&th,0,start_routine,0); - pthread_join(th, 0); - pthread_attr_init(&attr); - pthread_cleanup_push(routine, 0); - pthread_cleanup_pop(0); ], - [ax_pthread_ok=yes]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include + static void routine(void *a) { a = 0; } + static void *start_routine(void *a) { return a; }], + [pthread_t th; pthread_attr_t attr; + pthread_create(&th, 0, start_routine, 0); + pthread_join(th, 0); + pthread_attr_init(&attr); + pthread_cleanup_push(routine, 0); + pthread_cleanup_pop(0) /* ; */])], + [ax_pthread_ok=yes], + []) LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" - AC_MSG_RESULT($ax_pthread_ok) + AC_MSG_RESULT([$ax_pthread_ok]) if test "x$ax_pthread_ok" = xyes; then break; fi @@ -230,55 +251,82 @@ CFLAGS="$CFLAGS $PTHREAD_CFLAGS" # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. - AC_MSG_CHECKING([for joinable pthread attribute]) - attr_name=unknown - for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do - AC_TRY_LINK([#include ], [int attr=$attr; return attr;], - [attr_name=$attr; break]) - done - AC_MSG_RESULT($attr_name) + AC_MSG_CHECKING([for joinable pthread attribute]) + attr_name=unknown + for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [int attr = $attr; return attr /* ; */])], + [attr_name=$attr; break], + []) + done + AC_MSG_RESULT([$attr_name]) if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then - AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name, + AC_DEFINE_UNQUOTED([PTHREAD_CREATE_JOINABLE], [$attr_name], [Define to necessary symbol if this constant uses a non-standard name on your system.]) fi AC_MSG_CHECKING([if more special flags are required for pthreads]) flag=no - case "${host_cpu}-${host_os}" in - *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";; - *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";; + case ${host_os} in + aix* | freebsd* | darwin*) flag="-D_THREAD_SAFE";; + osf* | hpux*) flag="-D_REENTRANT";; + solaris*) + if test "$GCC" = "yes"; then + flag="-D_REENTRANT" + else + # TODO: What about Clang on Solaris? + flag="-mt -D_REENTRANT" + fi + ;; esac - AC_MSG_RESULT(${flag}) + AC_MSG_RESULT([$flag]) if test "x$flag" != xno; then PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS" fi + AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT], + [ax_cv_PTHREAD_PRIO_INHERIT], [ + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], + [[int i = PTHREAD_PRIO_INHERIT;]])], + [ax_cv_PTHREAD_PRIO_INHERIT=yes], + [ax_cv_PTHREAD_PRIO_INHERIT=no]) + ]) + AS_IF([test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes"], + [AC_DEFINE([HAVE_PTHREAD_PRIO_INHERIT], [1], [Have PTHREAD_PRIO_INHERIT.])]) + LIBS="$save_LIBS" CFLAGS="$save_CFLAGS" - # More AIX lossage: must compile with xlc_r or cc_r - if test x"$GCC" != xyes; then - AC_CHECK_PROGS(PTHREAD_CC, xlc_r cc_r, ${CC}) - else - PTHREAD_CC=$CC - fi -else - PTHREAD_CC="$CC" + # More AIX lossage: compile with *_r variant + if test "x$GCC" != xyes; then + case $host_os in + aix*) + AS_CASE(["x/$CC"], + [x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6], + [#handle absolute path differently from PATH based program lookup + AS_CASE(["x$CC"], + [x/*], + [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])], + [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])]) + ;; + esac + fi fi -AC_SUBST(PTHREAD_LIBS) -AC_SUBST(PTHREAD_CFLAGS) -AC_SUBST(PTHREAD_CC) +test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" + +AC_SUBST([PTHREAD_LIBS]) +AC_SUBST([PTHREAD_CFLAGS]) +AC_SUBST([PTHREAD_CC]) # Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: if test x"$ax_pthread_ok" = xyes; then - ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1]) + ifelse([$1],,[AC_DEFINE([HAVE_PTHREAD],[1],[Define if you have POSIX threads libraries and header files.])],[$1]) : else ax_pthread_ok=no $2 fi -AC_LANG_RESTORE +AC_LANG_POP ])dnl AX_PTHREAD - diff -Nru osm2pgsql-0.82.0/Makefile.am osm2pgsql-0.88.1/Makefile.am --- osm2pgsql-0.82.0/Makefile.am 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/Makefile.am 2015-08-15 05:28:11.000000000 +0000 @@ -1,16 +1,114 @@ ACLOCAL_AMFLAGS = -I m4 - -DIST_SUBDIRS = legacy +AUTOMAKE_OPTIONS = subdir-objects bin_PROGRAMS = osm2pgsql nodecachefilereader +noinst_LTLIBRARIES = libosm2pgsql.la -osm2pgsql_SOURCES = build_geometry.cpp input.c middle.h middle-ram.h output-gazetteer.h output-pgsql.c rb.c sanitizer.h text-tree.h build_geometry.h input.h middle-pgsql.c osm2pgsql.c output.h output-pgsql.h rb.h sprompt.c UTF8sanitizer.c expire-tiles.c keyvals.c middle-pgsql.h osmtypes.h output-null.c parse-o5m.c parse-o5m.h parse-primitive.c parse-primitive.h parse-xml2.c parse-xml2.h pgsql.c reprojection.c sprompt.h expire-tiles.h keyvals.h middle-ram.c output-gazetteer.c output-null.h pgsql.h reprojection.h text-tree.c node-ram-cache.c wildcmp.c node-ram-cache.h node-persistent-cache.c node-persistent-cache.h binarysearcharray.c binarysearcharray.h +osm2pgsql_SOURCES = osm2pgsql.cpp -nodecachefilereader_SOURCES = node-persistent-cache-reader.c node-persistent-cache.c node-ram-cache.c binarysearcharray.c +osm2pgsql_LDADD = libosm2pgsql.la +libosm2pgsql_la_SOURCES = \ + UTF8sanitizer.cpp \ + expire-tiles.cpp \ + geometry-builder.cpp \ + geometry-processor.cpp \ + id-tracker.cpp \ + input.cpp \ + middle.cpp \ + middle-pgsql.cpp \ + middle-ram.cpp \ + node-persistent-cache.cpp \ + node-ram-cache.cpp \ + options.cpp \ + osmdata.cpp \ + output.cpp \ + output-gazetteer.cpp \ + output-multi.cpp \ + output-null.cpp \ + output-pgsql.cpp \ + parse.cpp \ + parse-o5m.cpp \ + parse-pbf.cpp \ + parse-xml2.cpp \ + pgsql.cpp \ + processor-line.cpp \ + processor-point.cpp \ + processor-polygon.cpp \ + reprojection.cpp \ + sprompt.cpp \ + table.cpp \ + taginfo.cpp \ + tagtransform.cpp \ + util.cpp \ + wildcmp.cpp + +nodecachefilereader_SOURCES = node-persistent-cache-reader.cpp +nodecachefilereader_LDADD = libosm2pgsql.la + +check_PROGRAMS = \ + tests/test-parse-xml2 \ + tests/test-middle-ram \ + tests/test-middle-pgsql \ + tests/test-middle-flat \ + tests/test-output-multi-tags \ + tests/test-output-multi-line \ + tests/test-output-multi-line-storage \ + tests/test-output-multi-point \ + tests/test-output-multi-point-multi-table \ + tests/test-output-multi-polygon \ + tests/test-output-multi-poly-trivial \ + tests/test-output-pgsql \ + tests/test-output-pgsql-z_order \ + tests/test-output-pgsql-tablespace \ + tests/test-pgsql-escape \ + tests/test-parse-options \ + tests/test-expire-tiles + +tests_test_parse_xml2_SOURCES = tests/test-parse-xml2.cpp +tests_test_parse_xml2_LDADD = libosm2pgsql.la +tests_test_middle_ram_SOURCES = tests/test-middle-ram.cpp tests/middle-tests.cpp +tests_test_middle_ram_LDADD = libosm2pgsql.la +tests_test_middle_pgsql_SOURCES = tests/test-middle-pgsql.cpp tests/middle-tests.cpp tests/common-pg.cpp +tests_test_middle_pgsql_LDADD = libosm2pgsql.la +tests_test_middle_flat_SOURCES = tests/test-middle-flat.cpp tests/middle-tests.cpp tests/common-pg.cpp +tests_test_middle_flat_LDADD = libosm2pgsql.la +tests_test_output_multi_tags_SOURCES = tests/test-output-multi-tags.cpp tests/common-pg.cpp +tests_test_output_multi_tags_LDADD = libosm2pgsql.la +tests_test_output_multi_line_SOURCES = tests/test-output-multi-line.cpp tests/common-pg.cpp +tests_test_output_multi_line_LDADD = libosm2pgsql.la +tests_test_output_multi_line_storage_SOURCES = tests/test-output-multi-line-storage.cpp tests/common-pg.cpp +tests_test_output_multi_line_storage_LDADD = libosm2pgsql.la +tests_test_output_multi_point_SOURCES = tests/test-output-multi-point.cpp tests/common-pg.cpp +tests_test_output_multi_point_LDADD = libosm2pgsql.la +tests_test_output_multi_point_multi_table_SOURCES = tests/test-output-multi-point-multi-table.cpp tests/common-pg.cpp +tests_test_output_multi_point_multi_table_LDADD = libosm2pgsql.la +tests_test_output_multi_polygon_SOURCES = tests/test-output-multi-polygon.cpp tests/common-pg.cpp +tests_test_output_multi_polygon_LDADD = libosm2pgsql.la +tests_test_output_multi_poly_trivial_SOURCES = tests/test-output-multi-poly-trivial.cpp tests/common-pg.cpp +tests_test_output_multi_poly_trivial_LDADD = libosm2pgsql.la +tests_test_output_pgsql_SOURCES = tests/test-output-pgsql.cpp tests/common-pg.cpp +tests_test_output_pgsql_LDADD = libosm2pgsql.la +tests_test_output_pgsql_tablespace_SOURCES = tests/test-output-pgsql-tablespace.cpp tests/common-pg.cpp +tests_test_output_pgsql_tablespace_LDADD = libosm2pgsql.la +tests_test_output_pgsql_z_order_SOURCES = tests/test-output-pgsql-z_order.cpp tests/common-pg.cpp +tests_test_output_pgsql_z_order_LDADD = libosm2pgsql.la +tests_test_pgsql_escape_SOURCES = tests/test-pgsql-escape.cpp +tests_test_pgsql_escape_LDADD = libosm2pgsql.la +tests_test_parse_options_SOURCES = tests/test-parse-options.cpp +tests_test_parse_options_LDADD = libosm2pgsql.la +tests_test_expire_tiles_SOURCES = tests/test-expire-tiles.cpp +tests_test_expire_tiles_LDADD = libosm2pgsql.la + +MOSTLYCLEANFILES = tests/test_middle_flat.flat.nodes.bin tests/test_output_pgsql_area_way.flat.nodes.bin + +TESTS = $(check_PROGRAMS) tests/regression-test.sh +TEST_EXTENSIONS = .sh +SH_LOG_COMPILER = sh if READER_PBF -osm2pgsql_SOURCES += parse-pbf.c parse-pbf.h fileformat.pb-c.c fileformat.pb-c.h osmformat.pb-c.c osmformat.pb-c.h +osm2pgsql_SOURCES += parse-pbf.hpp fileformat.pb-c.h osmformat.pb-c.h +libosm2pgsql_la_SOURCES += parse-pbf.cpp fileformat.pb-c.c osmformat.pb-c.c fileformat.pb-c.c: protobuf/fileformat.proto $(AM_V_GEN) $(PROTOC_C) --proto_path=protobuf --c_out=. $< @@ -34,14 +132,37 @@ fileformat.pb-c.c fileformat.pb-c.h \ osmformat.pb-c.c osmformat.pb-c.h +CLEANFILES = \ + fileformat.pb-c.c fileformat.pb-c.h \ + osmformat.pb-c.c osmformat.pb-c.h + endif osm2pgsqldir = $(datadir)/osm2pgsql -AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @PROTOBUF_C_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' -AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -Igeos-fallback +AM_CFLAGS = @PTHREAD_CFLAGS@ @LFS_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ @PROTOBUF_C_CFLAGS@ @ZLIB_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -DVERSION='"@PACKAGE_VERSION@"' @LUA_INCLUDE@ +AM_CPPFLAGS = @PTHREAD_CFLAGS@ @POSTGRESQL_CPPFLAGS@ @XML2_CFLAGS@ @BZIP2_CFLAGS@ @GEOS_CFLAGS@ @PROJ_CFLAGS@ -DOSM2PGSQL_DATADIR='"$(osm2pgsqldir)"' -Igeos-fallback @LUA_INCLUDE@ @BOOST_CPPFLAGS@ -AM_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML2_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_C_LDFLAGS@ @PROTOBUF_C_LIBS@ +GLOBAL_LDFLAGS = @PTHREAD_CFLAGS@ @ZLIB_LDFLAGS@ @ZLIB_LIBS@ @POSTGRESQL_LDFLAGS@ @POSTGRESQL_LIBS@ @XML2_LDFLAGS@ @BZIP2_LDFLAGS@ @BZIP2_LIBS@ @GEOS_LDFLAGS@ @GEOS_LIBS@ @PROJ_LDFLAGS@ @PROJ_LIBS@ @PROTOBUF_C_LDFLAGS@ @PROTOBUF_C_LIBS@ -L/usr/lib/x86_64-linux-gnu @LUA_LIB@ @BOOST_LDFLAGS@ @BOOST_FILESYSTEM_LIB@ @BOOST_SYSTEM_LIB@ @BOOST_THREAD_LIB@ +osm2pgsql_LDADD += $(GLOBAL_LDFLAGS) +tests_test_parse_xml2_LDADD += $(GLOBAL_LDFLAGS) +tests_test_middle_ram_LDADD += $(GLOBAL_LDFLAGS) +tests_test_middle_pgsql_LDADD += $(GLOBAL_LDFLAGS) +tests_test_middle_flat_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_multi_tags_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_multi_line_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_multi_line_storage_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_multi_point_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_multi_point_multi_table_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_multi_polygon_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_multi_poly_trivial_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_pgsql_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_pgsql_tablespace_LDADD += $(GLOBAL_LDFLAGS) +tests_test_output_pgsql_z_order_LDADD += $(GLOBAL_LDFLAGS) +tests_test_pgsql_escape_LDADD += $(GLOBAL_LDFLAGS) +tests_test_parse_options_LDADD += $(GLOBAL_LDFLAGS) +tests_test_expire_tiles_LDADD += $(GLOBAL_LDFLAGS) +nodecachefilereader_LDADD += $(GLOBAL_LDFLAGS) osm2pgsql_DATA = default.style 900913.sql @@ -63,3 +184,5 @@ distclean-local: @rm -f $(PACKAGE).spec @rm -f config.nice + +test: check diff -Nru osm2pgsql-0.82.0/mapnik-osm-updater.sh osm2pgsql-0.88.1/mapnik-osm-updater.sh --- osm2pgsql-0.82.0/mapnik-osm-updater.sh 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/mapnik-osm-updater.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,784 +0,0 @@ -#!/bin/bash - -export osm_username="osm" -export database_name="gis" -export planet_dir="/home/$osm_username/osm/planet" -export planet_file="$planet_dir/planet.osm.bz2" -export sql_dump="$planet_dir/planet.osm.sql.bz2" -export log_dir=/var/log - -export geoinfodb_file="/usr/share/icons/map-icons/geoinfo.db" -export osmdb_file="/usr/share/gpsdrive/osm.db" - -export osm2pgsql_cmd=`which osm2pgsql` -test -x "$osm2pgsql_cmd" || echo "Missing osm2pgsql in PATH" -test -x "$osm2pgsql_cmd" || osm2pgsql_cmd="$HOME/svn.openstreetmap.org/applications/utils/export/osm2pgsql/osm2pgsql" -test -x "$osm2pgsql_cmd" || echo "Missing osm2pgsql" - -export cmd_osm2poidb=`which osm2poidb` -test -x "$cmd_osm2poidb" || echo "Missing osm2poidb in PATH" -test -x "$cmd_osm2poidb" || cmd_osm2poidb="`dirname $0`/../osm2poidb/build/osm2poidb" -test -x "$cmd_osm2poidb" || cmd_osm2poidb="$HOME/svn.openstreetmap.org/applications/utils/export/osm2poidb/build/osm2poidb" -test -x "$cmd_osm2poidb" || echo "Missing osm2poidb" - -osm_planet_mirror_cmd=`which osm-planet-mirror` -test -x "$osm_planet_mirror_cmd" || echo "Missing planet-mirror.pl in PATH" -test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd="`dirname $0`/../../planet-mirror/planet-mirror.pl" -test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd="$HOME/svn.openstreetmap.org/applications/utils/planet-mirror/planet-mirror.pl" -test -x "$osm_planet_mirror_cmd" || osm_planet_mirror_cmd="`dirname ../../planet-mirror/planet-mirror.pl`" -test -x "$osm_planet_mirror_cmd" || echo "Missing planet-mirror.pl" - -test -n "$1" || help=1 -quiet=" -q " -verbose=1 - -for arg in "$@" ; do - case $arg in - --all-planet) # Do all the creation steps listed below from planet file - create_osm_user=1 - mirror=1 - check_newer_planet= - drop=1 - create_db=1 - db_table_create=1 - create_db=1 - create_db_user=1 - db_add_900913=1 - db_add_spatial_ref_sys=1 - grant_all_rights_to_user_osm=1 - planet_fill=1 - db_add_gpsdrive_poitypes=1 - create_db_users=${create_db_users:-*} - grant_db_users=${grant_db_users:-*} - ;; - - --all-planet-geofabrik=\?) # Use Planet Extract from Frederics GeoFabrik.de Page as planet File and import - # Use ? for a list of possible files - dir_country=${arg#*=} - country=`basename $dir_country` - planet_file="$planet_dir/${country}.osm.bz2" - mirror_geofabrik=${dir_country} - mirror= - ;; - - --all-planet-geofabrik=*) # Use Planet Extract from Frederics GeoFabrik.de Page as planet File and import - # Use ? for a list of possible files - # Example: europe/germany/baden-wuerttemberg - dir_country=${arg#*=} - country=`basename $dir_country` - planet_file="$planet_dir/${country}.osm.bz2" - mirror_geofabrik=${dir_country} - create_osm_user=1 - mirror= - check_newer_planet= - drop=1 - create_db=1 - db_table_create=1 - db_add_900913=1 - db_add_spatial_ref_sys=1 - create_db_user=1 - grant_all_rights_to_user_osm=1 - planet_fill=1 - db_add_gpsdrive_poitypes=1 - create_db_users=${create_db_users:-*} - grant_db_users=${grant_db_users:-*} - ;; - - --all-planet-update) # Do all the creation steps listed below from planet file with up to date checking - create_osm_user=1 - mirror=1 - check_newer_planet=1 - drop=1 - create_db=1 - db_add_900913=1 - db_add_spatial_ref_sys=1 - create_db_user=1 - grant_all_rights_to_user_osm=1 - planet_fill=1 - db_add_gpsdrive_poitypes=1 - create_db_users=${create_db_users:-*} - grant_db_users=${grant_db_users:-*} - ;; - - --all-from-dump) # Do all the creation steps listed below - # from planet-dump file - # !!! all-from-dump is not completely tested yet - create_osm_user=1 - mirror_dump=1 - drop=1 - create_db=1 - db_add_900913=1 - db_add_spatial_ref_sys=1 - create_db_user=1 - grant_all_rights_to_user_osm=1 - create_db_users=${create_db_users:-*} - fill_from_dump="$sql_dump" - grant_db_users=${grant_db_users:-*} - db_add_gpsdrive_poitypes=1 - ;; - - --all-create) # Do all the creation steps listed below only no data - # import and no planet mirroring - create_osm_user=1 - drop=1 - create_db=1 - db_add_900913=1 - db_add_spatial_ref_sys=1 - create_db_user=1 - grant_all_rights_to_user_osm=1 - create_db_users=${create_db_users:-*} - grant_db_users=${grant_db_users:-*} - ;; - - --create-osm-user) # create the osm-user needed - # This means creating a user 'osm' and his home directory - # with useradd, mkdir, chmod and chown - create_osm_user=1 - ;; - - --mirror) # mirror planet File (http://planet.openstreetmap.org/) - mirror=1 - ;; - - --no-mirror) # do not mirror planet File - mirror= - ;; - - --check-newer-planet) # Check if Planet File is newer then stampfile. - # If yes: Continue - check_newer_planet=1 - ;; - - --drop) # drop the old Database (gis) and Database-user (osm) - drop=1 - ;; - - --create-db) # create the database (gis) - # with this command only the database is created, - # but no tables inside it - create_db=1 - ;; - - --create-db-user) # create the database-user (osm) - create_db_user=1 - ;; - - --grant-all2osm-user) # grant all rights for the database to the DB-User osm - grant_all_rights_to_user_osm=1 - ;; - - --create-db-users=*) #Create a Database user for all users specified. - # To create a db-user for all available system-user - # specify *. (Except root)) - create_db_users=${arg#*=} - ;; - - --grant-db-users=*) # Grant database-users all rights (including write, ...) - # to the gis Database !!! This has to be changed in the - # future, normally only the osm user needs update rights - grant_db_users=${arg#*=} - ;; - - --add-gpsdrive-types) # add GpsDrive POI-Types to points table - db_add_gpsdrive_poitypes=1 - ;; - - --planet-fill) # fill database from planet File - planet_fill=1 - ;; - - --mirror-dump) # mirror the planet.sql dump File - mirror_dump=1 - ;; - - --no-mirror-dump) # Do not mirror the planet.sql dump File - mirror_dump= - ;; - - --fill-from-dump=*) # fill database from Dump File - fill_from_dump=${arg#*=} - ;; - - --mapnik-dump=*) # Dump Content of Mapnik Database to a File (.sql|.sql.bz)) - postgis_mapnik_dump=${arg#*=} - ;; - - --db-table-create) # Create tables in Database with osm2pgsql - db_table_create=1 - ;; - - --db-add-srid-900913) # Add SRID 900913 - db_add_900913=1 - ;; - - --db-add-spatial_ref_sys) # Add SRIDs to spatial_ref_sys - db_add_spatial_ref_sys=1 - ;; - - --count-db) # Count entries in Database. This is to check - # if the database really contains entries - # if you set an empty user with the option osm_username='' - # the current user is used - count_db=1 - ;; - - -h) - help=1 - ;; - - --help) - help=1 - ;; - - -help) - help=1 - ;; - - --debug) # switch on debugging - debug=1 - verbose=1 - quiet="" - ;; - - -debug) - debug=1 - verbose=1 - quiet="" - ;; - - - --nv) # be a little bit less verbose - verbose='' - ;; - - --planet-dir=*) # define Directory for Planet-File - planet_dir=${arg#*=} - planet_file="$planet_dir/planet.osm.bz2" - ;; - - --planet-file=*) # define Planet-File including Directory - planet_file=${arg#*=} - ;; - - --poi-file=*) # define POI Database file including Directory - osmdb_file=${arg#*=} - ;; - - --geoinfo-file=*) # define geoinfo database file containing poi-types - geoinfodb_file=${arg#*=} - ;; - - --osm-username=*) # Define username to use for DB creation and planet - # download - # !! You shouldn't use your username or root as the - # !! download and install user. - # This username is the download and install user. - # The osm-user normally only should have the planet files - # in hishome directory and nothing else. By default - # the osm-username is 'osm' - osm_username=${arg#*=} - - if [ "$osm_username" = "$USER" ] ; then - echo - echo "!!!!!! ERROR: Don't use your own login account as the osm_username!!" 1>&2 - echo - exit 1 - fi - - if [ "$osm_username" = "root" ] ; then - echo - echo "!!!!!! ERROR: Don't use the root account as the osm_username!!" 1>&2 - echo - exit 1 - fi - - planet_dir="/home/$osm_username/osm/planet" - planet_file="$planet_dir/planet.osm.bz2" - ;; - - --osm2pgsql-cmd=*) # The path to the osm2pgsql command - # It can be found at - # svn.openstreetmap.org/applications/utils/export/osm2pgsql/ - # and has to be compiled. Alternatively you can install - # the Debian Package openstreetmap-utils - osm2pgsql_cmd=${arg#*=} - if ! [ -x "$osm2pgsql_cmd" ]; then - echo "!!!!!! ERROR: Cannot execute '$osm2pgsql_cmd'" 1>&2 - exit -1 - fi - ;; - - --database-name=*) # use this name for the database default is 'gis' - database_name=${arg#*=} - ;; - - *) - echo "" - echo "!!!!!!!!! Unknown option $arg" - echo "" - help=1 - ;; - esac -done - -if [ -n "$help" ] ; then - # extract options from case commands above - options=`grep -E -e esac -e '\s*--.*\).*#' $0 | sed '/esac/,$d;s/.*--/ [--/; s/=\*)/=val]/; s/)[\s ]/]/; s/#.*\s*//; s/[\n/]//g;'` - options=`for a in $options; do echo -n " $a" ; done` - echo "$0 $options" - echo " -!!! Warning: This Script is for now a quick hack to make setting up -!!! Warning: My databases easier. Please check if it really works for you!! -!!! Warning: Especially when using different Database names or username, ... -!!! Warning: not every combination of values except the default is tested. - - This script tries to install the mapnik database. - For this it first creates a new user osm on the system - and mirrors the current planet to his home directory. - Then this planet is imported into the postgis Database from a - newly created user named osm - - This script uses sudo. So you either have to have sudo right or you'll - have to start the script as root. The users needed will be postgres and osm - " - # extract options + description from case commands above - grep -E -e esac -e '--.*\).*#' -e '^[\t\s ]+#' $0 | \ - grep -v /bin/bash | sed '/esac/,$d;s/.*--/ --/;s/=\*)/=val/;s/)//;s/#//;s/\\//;' - exit; -fi - - -if [ -n "$osm_username" ] ; then - sudo_cmd="sudo -u $osm_username" -else - sudo_cmd='' -fi - -export import_stamp_file=${log_dir}/osm2pgsql_postgis-$database_name.stamp -export import_log=${log_dir}/osm2pgsql_postgis-$database_name.log - - -if [ -n "$debug" ] ; then - echo "Planet File: `ls -l $planet_file`" - echo "Import Stamp : `ls -l $import_stamp_file`" -fi - - -############################################ -# Create a user on the system -############################################ -if [ -n "$create_osm_user" ] ; then - test -n "$verbose" && echo "----- Check if we already have an user '$osm_username'" - - if ! id "$osm_username" >/dev/null; then - echo "create '$osm_username' User" - useradd "$osm_username" - fi - - mkdir -p "/home/$osm_username/osm/planet" - # The user itself should be allowed to read/write all his own files - # in the ~/osm/ Directory - chown "$osm_username" "/home/$osm_username" - chown -R "$osm_username" "/home/$osm_username/osm" - chmod +rwX "/home/$osm_username" - chmod -R +rwX "/home/$osm_username/osm" - - # Everyone on the system is allowed to read the planet.osm Files - chmod -R a+rX "/home/$osm_username/osm" -fi - - -############################################ -# Mirror the planet-dump File for Europe -############################################ -if [ -n "$mirror_geofabrik" ] ; then - geofabrik_basedir="http://download.geofabrik.de/osm" - if [ "$mirror_geofabrik" = "?" ]; then - - echo "Retreiving available planet extracts from GeoFabrik ..." - # Find all Subdirs in the first 3 levels - wget_out=`wget --no-convert-links -q --level=0 -O - "http://download.geofabrik.de/osm" | grep DIR | grep -v -i Parent ` - sub_dirs=`echo "$wget_out" | perl -ne 'm,href="(.*)/",;print "$1 "'` - - for level in 1 2 3; do - for sub_dir in $sub_dirs ; do - #echo "Get dirs in Subdir: $sub_dir" - wget_out=`wget -q --level=0 -O - "$geofabrik_basedir/$sub_dir" | grep 'DIR' | grep -v Parent ` - new_dirs="$new_dirs `echo "$wget_out" | perl -ne 'm,href="(.*)/", && print "'$sub_dir'/$1 "'`" - # echo "WGET: '$wget_out'" - done - sub_dirs="$sub_dirs $new_dirs" - done - sub_dirs=`for dir in $sub_dirs; do echo $dir; done | sort -u` - - - # Printout content of all $sub_dirs - - echo "Possible Values are:" - for sub_dir in "" $sub_dirs ; do - wget -q --level=0 -O - "$geofabrik_basedir/$sub_dir" | grep 'OpenStreetMap data' | \ - perl -ne 'm/.*href="([^"]+)\.osm.bz2"/;print " '$sub_dir/'$1\n"' - done - exit 1 - fi - planet_source_file="${geofabrik_basedir}/${mirror_geofabrik}.osm.bz2" - if [ -n "$mirror" ] ; then - test -n "$verbose" && echo "----- Mirroring planet File $planet_source_file" - wget -v --mirror "$planet_source_file" \ - --no-directories --directory-prefix=$planet_dir/ - fi -fi - - -############################################ -# Mirror the newest planet File from planet.openstreetmap.org -############################################ -if [ -n "$mirror" ] ; then - test -n "$verbose" && echo "----- Mirroring planet File" - if ! [ -x "$osm_planet_mirror_cmd" ]; then - echo "!!!!!! ERROR: Cannot execute '$osm_planet_mirror_cmd'" 1>&2 - exit -1 - fi - if ! $sudo_cmd $osm_planet_mirror_cmd -v -v --planet-dir=$planet_dir ; then - echo "!!!!!! ERROR: Cannot Mirror Planet File" 1>&2 - exit 1 - fi - if ! [ -s $planet_file ] ; then - echo "!!!!!! ERROR: File $planet_file is missing" - exit -1 - fi - - -fi - -############################################ -# Check if Planet File is newer than import Stamp -############################################ -if [ -n "$check_newer_planet" ] ; then - if [ "$planet_file" -nt "$import_stamp_file" ] ; then - if [ -n "$verbose" ] ; then - echo "----- New File needs updating" - echo "Planet File: `ls -l $planet_file`" - echo "Import Stamp : `ls -l $import_stamp_file`" - fi - else - echo "Postgis Database already Up To Date" - echo "`ls -l $import_stamp_file`" - exit 0 - fi -fi - -############################################ -# Drop the old Database and Database-user -############################################ -if [ -n "$drop" ] ; then - test -n "$verbose" && echo "----- Drop complete Database '$database_name' and user '$osm_username'" - echo "CHECKPOINT" | sudo -u postgres psql $quiet - sudo -u postgres dropdb $quiet -Upostgres "$database_name" - sudo -u postgres dropuser $quiet -Upostgres "$osm_username" -fi - -############################################ -# Create db -############################################ -if [ -n "$create_db" ] ; then - test -n "$verbose" && echo - test -n "$verbose" && echo "----- Create Database '$database_name'" - if ! sudo -u postgres createdb -Upostgres $quiet -EUTF8 "$database_name"; then - echo "!!!!!! ERROR: Creation of '$database_name' Failed" - exit -1 - fi - if ! sudo -u postgres createlang plpgsql "$database_name"; then - echo "!!!!!! ERROR: Creation Failed" - exit -1 - fi - - lwpostgis="/usr/share/postgresql-8.4-postgis/lwpostgis.sql" - test -s $lwpostgis || lwpostgis="/usr/share/postgresql-8.3-postgis/lwpostgis.sql" - test -s $lwpostgis || lwpostgis="/usr/share/postgresql-8.2-postgis/lwpostgis.sql" - test -s $lwpostgis || lwpostgis="`ls /usr/share/postgresql-*-postgis/lwpostgis.sql| sort -n | head 1`" - if [ ! -s $lwpostgis ] ; then - echo "!!!!!! ERROR: Cannot find $lwpostgis" - exit -1 - fi - if sudo -u postgres psql $quiet -Upostgres "$database_name" <${lwpostgis} ; then - echo "Enabling spacial Extentions done with '$lwpostgis'" - else - echo "!!!!!! ERROR: Creation with '$lwpostgis' Failed" - exit -1 - fi -fi - -############################################ -# Create db-user -############################################ -if [ -n "$create_db_user" ] ; then - test -n "$verbose" && echo "----- Create Database-user '$osm_username'" - sudo -u postgres createuser -Upostgres $quiet -S -D -R "$osm_username" || exit -1 -fi - -if [ -n "$grant_all_rights_to_user_osm" ] ; then - test -n "$verbose" && echo - test -n "$verbose" && echo "----- Grant rights on Database '$database_name' for '$osm_username'" - ( - echo "GRANT ALL ON SCHEMA PUBLIC TO \"$osm_username\";" - echo "GRANT ALL on geometry_columns TO \"$osm_username\";" - echo "GRANT ALL on spatial_ref_sys TO \"$osm_username\";" - echo "GRANT ALL ON SCHEMA PUBLIC TO \"$osm_username\";" - ) | sudo -u postgres psql $quiet -Upostgres "$database_name" -fi - -############################################ -# Create a Database user for all users specified (*) or available on the system. Except root -############################################ -if [ -n "$create_db_users" ] ; then - - if [ "$create_db_users" = "*" ] ; then - echo "Create DB User for every USER" - create_db_users='' - # try to see if all users above uid=1000 are interesting - all_users=`cat /etc/passwd | sed 's/:/ /g' | while read user pwd uid rest ; do test "$uid" -ge "1000" || continue; echo $user; done` - echo "all_users: $all_users" - for user in $all_users ; do - echo $user | grep -q -e root && continue - echo $user | grep -q -e "$osm_username" && continue - echo $user | grep -q -e "nobody" && continue - echo "$create_db_users" | grep -q " $user " && continue - create_db_users=" $create_db_users $user " - done - fi - -# This is not good; this probably broke my postgres installation -# dpkg --purge postgresql-8.2 -# Stopping PostgreSQL 8.2 database server: main* Error: The cluster is owned by user id 107 which does not exist any more -# apt-get -f install postgresql-8.2 -# Starting PostgreSQL 8.2 database server: main* Error: The cluster is owned by user id 107 which does not exist any more -#if false ; then - for user in $create_db_users; do - echo " Create DB User for $user" - sudo -u postgres createuser $quiet -Upostgres --no-superuser --no-createdb --no-createrole "$user" - done -#fi - -fi - -############################################ -# Create Database tables with osm2pgsql -############################################ -if [ -n "$db_table_create" ] ; then - if ! [ -x "$osm2pgsql_cmd" ]; then - echo "!!!!!! ERROR: Cannot execute '$osm2pgsql_cmd'" 1>&2 - exit -1 - fi - echo "" - echo "--------- Unpack and import $planet_file" - cd /usr/share/openstreetmap/ - $sudo_cmd $osm2pgsql_cmd --create "$database_name" -fi - - -############################################ -# Add SRID spatial_ref_sys -############################################ -if [ -n "$db_add_spatial_ref_sys" ] ; then - test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.4-postgis/spatial_ref_sys.sql" - test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.3-postgis/spatial_ref_sys.sql" - test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.2-postgis/spatial_ref_sys.sql" - test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-8.*-postgis/spatial_ref_sys.sql" - test -s "$srid_spatial_ref_sys" || srid_spatial_ref_sys="/usr/share/postgresql-*-postgis/spatial_ref_sys.sql" - if [ ! -s $srid_spatial_ref_sys ] ; then - echo "!!!!!! ERROR: Cannot find $srid_spatial_ref_sys" - exit -1 - fi - if sudo -u postgres psql $quiet -Upostgres "$database_name" <${srid_spatial_ref_sys} ; then - echo "Adding '$srid_spatial_ref_sys'" - else - echo "!!!!!! ERROR: Creation Failed" - exit -1 - fi -fi - - -############################################ -# Add SRID 900913 -############################################ -if [ -n "$db_add_900913" ] ; then - - test -s "$srid_900913" || srid_900913="`dirname $0`/900913.sql" - test -s "$srid_900913" || srid_900913="$HOME/svn.openstreetmap.org/applications/utils/export/osm2pgsql/900913.sql" - test -s "$srid_900913" || srid_900913="/usr/share/mapnik/900913.sql" - if [ ! -s $srid_900913 ] ; then - echo "!!!!!! ERROR: Cannot find $srid_900913" - exit -1 - fi - if sudo -u postgres psql $quiet -Upostgres "$database_name" <${srid_900913} ; then - echo "Adding '$srid_900913'" - else - echo "!!!!!! ERROR: Creation Failed" - exit -1 - fi -fi - - -############################################ -# Grant all rights on the gis Database to all system users or selected users in the system -############################################ -if [ -n "$grant_db_users" ] ; then - - if [ "$grant_db_users" = "*" ] ; then - echo "-------- GRANT Rights to every USER" - grant_db_users='' - for user in `users` ; do - echo "$user" | grep -q "root" && continue - echo " $grant_db_users " | grep -q " $user " && continue - grant_db_users="$grant_db_users $user" - done - fi - - test -n "$verbose" && echo "Granting rights to users: '$grant_db_users'" - - for user in $grant_db_users; do - echo "Granting all rights to user '$user' for Database '$database_name'" - ( - echo "GRANT ALL on geometry_columns TO \"$user\";" - echo "GRANT ALL ON SCHEMA PUBLIC TO \"$user\";" - echo "GRANT ALL on spatial_ref_sys TO \"$user\";" - echo "GRANT ALL on TABLE planet_osm_line TO \"$user\";" - echo "GRANT ALL on TABLE planet_osm_point TO \"$user\";" - echo "GRANT ALL on TABLE planet_osm_roads TO \"$user\";" - echo "GRANT ALL on TABLE planet_osm_polygon TO \"$user\";" - )| sudo -u postgres psql $quiet -Upostgres "$database_name" || true - done -fi - - -############################################ -# Fill Database from planet File -############################################ -if [ -n "$planet_fill" ] ; then - if ! [ -x "$osm2pgsql_cmd" ]; then - echo "!!!!!! ERROR: Cannot execute '$osm2pgsql_cmd'" 1>&2 - exit -1 - fi - echo "" - echo "--------- Unpack and import $planet_file" - echo "Import started: `date`" >>"$import_log" - cd /usr/share/openstreetmap/ - $sudo_cmd $osm2pgsql_cmd --database "$database_name" $planet_file - rc=$? - if [ "$rc" -gt "0" ]; then - echo "`date`: Import With Error $rc:" >> "$import_log" - echo "`ls -l $planet_file` import --> rc($rc)" >> "$import_log" - echo "!!!!!!!! ERROR while running '$sudo_cmd $osm2pgsql_cmd --database "$database_name" $planet_file'" - echo "Creation with for Database "$database_name" from planet-file '$planet_file' with '$osm2pgsql_cmd' Failed" - echo "see Logfile for more Information:" - echo "less $import_log" - exit -1 - fi - echo "`date`: Import Done: `ls -l $planet_file` import --> $rc" >> "$import_log" - echo "`date`: `ls -l $planet_file` import --> $rc" >>$import_stamp_file - touch --reference=$planet_file $import_stamp_file -fi - - -############################################ -# Create GpsDrive POI-Database -############################################ -if [ -n "$db_add_gpsdrive_poitypes" ] ; then - if ! [ -x "$cmd_osm2poidb" ]; then - echo "!!!!!! ERROR: Cannot execute gpsdrive_poitypes: '$cmd_osm2poidb'" 1>&2 - exit -1 - fi - echo "" - echo "--------- Create GpsDrive POI-Database $osmdb_file" - bunzip2 -c $planet_file | sudo $cmd_osm2poidb -w -f $geoinfodb_file -o $osmdb_file STDIN - rc=$? - if [ "$rc" -ne "0" ]; then - echo "!!!!!!! ERROR: cannot create POI Database" - exit -1 - fi -fi - - -############################################ -# Dump the complete Database -############################################ -if [ -n "$postgis_mapnik_dump" ] ; then - # get Database Content with Dump - postgis_mapnik_dump_dir=`dirname $postgis_mapnik_dump` - mkdir -p "$postgis_mapnik_dump_dir" - case "$postgis_mapnik_dump" in - *.bz2) - $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" | bzip2 >"$postgis_mapnik_dump" - ;; - *.gz) - $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" | gzip >"$postgis_mapnik_dump" - ;; - *) - $sudo_cmd pg_dump --data-only -U "$osm_username" "$database_name" >"$postgis_mapnik_dump" - ;; - esac - if [ "$?" -gt "0" ]; then - echo "Error While dumping Database" - fi -fi - -############################################ -# Mirror the planet-dump File from planet.openstreetmap.de -############################################ -if [ -n "$mirror_dump" ] ; then - test -n "$verbose" && echo "----- Mirroring planet-dump File" - wget -v --mirror http://planet.openstreetmap.de/planet.osm.sql.bz2 \ - --no-directories --directory-prefix=$planet_dir/ -fi - - -############################################ -# Fill Database from Dump File -############################################ -if [ -n "$fill_from_dump" ] ; then - echo "" - echo "--------- Import from Dump '$fill_from_dump'" - sudo -u postgres createdb -T template0 $database_name - case "$fill_from_dump" in - *.bz2) - test -n "$verbose" && echo "Uncompress File ..." - bzip2 -dc "$fill_from_dump" | $sudo_cmd psql $quiet "$database_name" - ;; - *.gz) - test -n "$verbose" && echo "Uncompress File ..." - gzip -dc "$fill_from_dump" | $sudo_cmd psql $quiet "$database_name" - ;; - *) - test -n "$verbose" && echo "Import uncompressed File ..." - $sudo_cmd psql $quiet "$database_name" <"$fill_from_dump" - ;; - esac - if [ "$?" -gt "0" ]; then - echo "Error While reding Dump into Database" - fi -fi - - -############################################ -# Check number of entries in Database -############################################ -if [ -n "$count_db" ] ; then - echo "" - echo "--------- Check Number of lines in Database '$database_name'" - - # Get the Table names - if [ -n "$osm_username" ]; then - table_owner=" AND tableowner ='$osm_username' "; - fi - table_names=`echo "SELECT tablename from pg_catalog.pg_tables where schemaname = 'public' $tableowner;" | \ - $sudo_cmd psql "$database_name" -h /var/run/postgresql | grep -E -e '^ planet'` - - echo "Counting entries in all Tables (" $table_names ")" - for table in $table_names; do - echo -n "Table $table = " - echo "SELECT COUNT(*) from $table;" | \ - $sudo_cmd psql gis -h /var/run/postgresql | grep -v -e count -e '------' -e '1 row' | head -1 - done -fi - diff -Nru osm2pgsql-0.82.0/middle.cpp osm2pgsql-0.88.1/middle.cpp --- osm2pgsql-0.82.0/middle.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/middle.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,14 @@ +#include "middle.hpp" +#include "middle-pgsql.hpp" +#include "middle-ram.hpp" + +#include + +boost::shared_ptr middle_t::create_middle(const bool slim) +{ + if(slim) + return boost::make_shared(); + else + return boost::make_shared(); +} + diff -Nru osm2pgsql-0.82.0/middle.h osm2pgsql-0.88.1/middle.h --- osm2pgsql-0.82.0/middle.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/middle.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,49 +0,0 @@ -/* Common middle layer interface */ - -/* Each middle layer data store must provide methods for - * storing and retrieving node and way data. - */ - -#ifndef MIDDLE_H -#define MIDDLE_H - -#include "osmtypes.h" - -struct keyval; -struct member; -struct output_options; - -struct middle_t { - int (*start)(const struct output_options *options); - void (*stop)(void); - void (*cleanup)(void); - void (*analyze)(void); - void (*end)(void); - void (*commit)(void); - - int (*nodes_set)(osmid_t id, double lat, double lon, struct keyval *tags); - int (*nodes_get_list)(struct osmNode *out, osmid_t *nds, int nd_count); - int (*nodes_delete)(osmid_t id); - int (*node_changed)(osmid_t id); - /* int (*nodes_get)(struct osmNode *out, osmid_t id);*/ - - int (*ways_set)(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags, int pending); - int (*ways_get)(osmid_t id, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr); - int (*ways_get_list)(osmid_t *ids, int way_count, osmid_t **way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr); - - int (*ways_done)(osmid_t id); - int (*ways_delete)(osmid_t id); - int (*way_changed)(osmid_t id); - - int (*relations_set)(osmid_t id, struct member *members, int member_count, struct keyval *tags); - /* int (*relations_get)(osmid_t id, struct member **members, int *member_count, struct keyval *tags); */ - int (*relations_done)(osmid_t id); - int (*relations_delete)(osmid_t id); - int (*relation_changed)(osmid_t id); - - /* void (*iterate_nodes)(int (*callback)(osmid_t id, struct keyval *tags, double node_lat, double node_lon)); */ - void (*iterate_ways)(int (*callback)(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists)); - void (*iterate_relations)(int (*callback)(osmid_t id, struct member *, int member_count, struct keyval *rel_tags, int exists)); -}; - -#endif diff -Nru osm2pgsql-0.82.0/middle.hpp osm2pgsql-0.88.1/middle.hpp --- osm2pgsql-0.82.0/middle.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/middle.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,79 @@ +/* Common middle layer interface */ + +/* Each middle layer data store must provide methods for + * storing and retrieving node and way data. + */ + +#ifndef MIDDLE_H +#define MIDDLE_H + +#include "osmtypes.hpp" + +#include +#include + +struct options_t; + +struct middle_query_t { + virtual ~middle_query_t() {} + + virtual int nodes_get_list(nodelist_t &out, const idlist_t nds) const = 0; + + virtual int ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const = 0; + + virtual int ways_get_list(const idlist_t &ids, idlist_t &way_ids, + multitaglist_t &tags, + multinodelist_t &nodes) const = 0; + + virtual int relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const = 0; + + virtual idlist_t relations_using_way(osmid_t way_id) const = 0; + + virtual boost::shared_ptr get_instance() const = 0; +}; + +struct middle_t : public middle_query_t { + static boost::shared_ptr create_middle(bool slim); + + virtual ~middle_t() {} + + virtual int start(const options_t *out_options_) = 0; + virtual void stop(void) = 0; + virtual void analyze(void) = 0; + virtual void end(void) = 0; + virtual void commit(void) = 0; + + virtual int nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags) = 0; + virtual int ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags) = 0; + virtual int relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags) = 0; + + struct pending_processor { + virtual ~pending_processor() {} + virtual void enqueue_ways(osmid_t id) = 0; + virtual void process_ways() = 0; + virtual void enqueue_relations(osmid_t id) = 0; + virtual void process_relations() = 0; + }; + + virtual void iterate_ways(pending_processor& pf) = 0; + virtual void iterate_relations(pending_processor& pf) = 0; + + virtual size_t pending_count() const = 0; + + const options_t* out_options; +}; + +struct slim_middle_t : public middle_t { + virtual ~slim_middle_t() {} + + virtual int nodes_delete(osmid_t id) = 0; + virtual int node_changed(osmid_t id) = 0; + + virtual int ways_delete(osmid_t id) = 0; + virtual int way_changed(osmid_t id) = 0; + + virtual int relations_delete(osmid_t id) = 0; + virtual int relation_changed(osmid_t id) = 0; +}; + +#endif diff -Nru osm2pgsql-0.82.0/middle-pgsql.c osm2pgsql-0.88.1/middle-pgsql.c --- osm2pgsql-0.82.0/middle-pgsql.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/middle-pgsql.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,1824 +0,0 @@ -/* Implements the mid-layer processing for osm2pgsql - * using several PostgreSQL tables - * - * This layer stores data read in from the planet.osm file - * and is then read by the backend processing code to - * emit the final geometry-enabled output formats -*/ - -#include "config.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_PTHREAD -#include -#endif - -#ifdef HAVE_SYS_WAIT_H -#include -#endif - -#ifdef HAVE_MMAP -#include -#ifndef MAP_ANONYMOUS -#ifdef MAP_ANON -#define MAP_ANONYMOUS MAP_ANON -#endif -#endif -#endif - -#include - -#include "osmtypes.h" -#include "middle.h" -#include "middle-pgsql.h" -#include "output-pgsql.h" -#include "node-ram-cache.h" -#include "node-persistent-cache.h" -#include "pgsql.h" - -struct progress_info { - time_t start; - time_t end; - int count; - int finished; -}; - -enum table_id { - t_node, t_way, t_rel -} ; - -struct table_desc { - const char *name; - const char *start; - const char *create; - const char *create_index; - const char *prepare; - const char *prepare_intarray; - const char *copy; - const char *analyze; - const char *stop; - const char *array_indexes; - - int copyMode; /* True if we are in copy mode */ - int transactionMode; /* True if we are in an extended transaction */ - PGconn *sql_conn; -}; - -static struct table_desc tables [] = { - { - /*table = t_node,*/ - .name = "%p_nodes", - .start = "BEGIN;\n", -#ifdef FIXED_POINT - .create = "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat int4 not null, lon int4 not null, tags text[]) {TABLESPACE %t};\n", - .prepare = "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", int4, int4, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n" -#else - .create = "CREATE %m TABLE %p_nodes (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, lat double precision not null, lon double precision not null, tags text[]) {TABLESPACE %t};\n", - .prepare = "PREPARE insert_node (" POSTGRES_OSMID_TYPE ", double precision, double precision, text[]) AS INSERT INTO %p_nodes VALUES ($1,$2,$3,$4);\n" -#endif - "PREPARE get_node (" POSTGRES_OSMID_TYPE ") AS SELECT lat,lon,tags FROM %p_nodes WHERE id = $1 LIMIT 1;\n" - "PREPARE delete_node (" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_nodes WHERE id = $1;\n", -.prepare_intarray = "PREPARE get_node_list(" POSTGRES_OSMID_TYPE "[]) AS SELECT id, lat, lon FROM %p_nodes WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[])", - .copy = "COPY %p_nodes FROM STDIN;\n", - .analyze = "ANALYZE %p_nodes;\n", - .stop = "COMMIT;\n" - }, - { - /*table = t_way,*/ - .name = "%p_ways", - .start = "BEGIN;\n", - .create = "CREATE %m TABLE %p_ways (id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, nodes " POSTGRES_OSMID_TYPE "[] not null, tags text[], pending boolean not null) {TABLESPACE %t};\n", - .create_index = "CREATE INDEX %p_ways_idx ON %p_ways (id) {TABLESPACE %i} WHERE pending;\n", -.array_indexes = "CREATE INDEX %p_ways_nodes ON %p_ways USING gin (nodes) {TABLESPACE %i};\n", - .prepare = "PREPARE insert_way (" POSTGRES_OSMID_TYPE ", " POSTGRES_OSMID_TYPE "[], text[], boolean) AS INSERT INTO %p_ways VALUES ($1,$2,$3,$4);\n" - "PREPARE get_way (" POSTGRES_OSMID_TYPE ") AS SELECT nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = $1;\n" - "PREPARE get_way_list (" POSTGRES_OSMID_TYPE "[]) AS SELECT id, nodes, tags, array_upper(nodes,1) FROM %p_ways WHERE id = ANY($1::" POSTGRES_OSMID_TYPE "[]);\n" - "PREPARE way_done(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_ways SET pending = false WHERE id = $1;\n" - "PREPARE pending_ways AS SELECT id FROM %p_ways WHERE pending;\n" - "PREPARE delete_way(" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_ways WHERE id = $1;\n", -.prepare_intarray = "PREPARE node_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_ways SET pending = true WHERE nodes && ARRAY[$1] AND NOT pending;\n", - .copy = "COPY %p_ways FROM STDIN;\n", - .analyze = "ANALYZE %p_ways;\n", - .stop = "COMMIT;\n" - }, - { - /*table = t_rel,*/ - .name = "%p_rels", - .start = "BEGIN;\n", - .create = "CREATE %m TABLE %p_rels(id " POSTGRES_OSMID_TYPE " PRIMARY KEY {USING INDEX TABLESPACE %i}, way_off int2, rel_off int2, parts " POSTGRES_OSMID_TYPE "[], members text[], tags text[], pending boolean not null) {TABLESPACE %t};\n", - .create_index = "CREATE INDEX %p_rels_idx ON %p_rels (id) {TABLESPACE %i} WHERE pending;\n", -.array_indexes = "CREATE INDEX %p_rels_parts ON %p_rels USING gin (parts) {TABLESPACE %i};\n", - .prepare = "PREPARE insert_rel (" POSTGRES_OSMID_TYPE ", int2, int2, " POSTGRES_OSMID_TYPE "[], text[], text[]) AS INSERT INTO %p_rels VALUES ($1,$2,$3,$4,$5,$6,false);\n" - "PREPARE get_rel (" POSTGRES_OSMID_TYPE ") AS SELECT members, tags, array_upper(members,1)/2 FROM %p_rels WHERE id = $1;\n" - "PREPARE rel_done(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = false WHERE id = $1;\n" - "PREPARE pending_rels AS SELECT id FROM %p_rels WHERE pending;\n" - "PREPARE delete_rel(" POSTGRES_OSMID_TYPE ") AS DELETE FROM %p_rels WHERE id = $1;\n", -.prepare_intarray = - "PREPARE node_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = true WHERE parts && ARRAY[$1] AND parts[1:way_off] && ARRAY[$1] AND NOT pending;\n" - "PREPARE way_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = true WHERE parts && ARRAY[$1] AND parts[way_off+1:rel_off] && ARRAY[$1] AND NOT pending;\n" - "PREPARE rel_changed_mark(" POSTGRES_OSMID_TYPE ") AS UPDATE %p_rels SET pending = true WHERE parts && ARRAY[$1] AND parts[rel_off+1:array_length(parts,1)] && ARRAY[$1] AND NOT pending;\n", - .copy = "COPY %p_rels FROM STDIN;\n", - .analyze = "ANALYZE %p_rels;\n", - .stop = "COMMIT;\n" - } -}; - -static const int num_tables = sizeof(tables)/sizeof(tables[0]); -static struct table_desc *node_table = &tables[t_node]; -static struct table_desc *way_table = &tables[t_way]; -static struct table_desc *rel_table = &tables[t_rel]; - -static int Append; - -const struct output_options *out_options; - -#define HELPER_STATE_UNINITIALIZED -1 -#define HELPER_STATE_FORKED -2 -#define HELPER_STATE_RUNNING 0 -#define HELPER_STATE_FINISHED 1 -#define HELPER_STATE_CONNECTED 2 -#define HELPER_STATE_FAILED 3 - -static int pgsql_connect(const struct output_options *options) { - int i; - /* We use a connection per table to enable the use of COPY */ - for (i=0; iconninfo); - - /* Check to see that the backend connection was successfully made */ - if (PQstatus(sql_conn) != CONNECTION_OK) { - fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn)); - return 1; - } - tables[i].sql_conn = sql_conn; - - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;"); - - if (tables[i].prepare) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare); - } - - if (tables[i].prepare_intarray) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray); - } - - } - return 0; -} - -static void pgsql_cleanup(void) -{ - int i; - - for (i=0; i (buflen-20) ) /* Almost overflowed? */ - { - buflen <<= 1; - buffer = realloc( buffer, buflen ); - - goto _restart; - } - first = 0; - } - - *ptr++ = '}'; - *ptr++ = 0; - - return buffer; -} - -/* Special escape routine for escaping strings in array constants: double quote, backslash,newline, tab*/ -static char *escape_tag( char *ptr, const char *in, int escape ) -{ - while( *in ) - { - switch(*in) - { - case '"': - if( escape ) *ptr++ = '\\'; - *ptr++ = '\\'; - *ptr++ = '"'; - break; - case '\\': - if( escape ) *ptr++ = '\\'; - if( escape ) *ptr++ = '\\'; - *ptr++ = '\\'; - *ptr++ = '\\'; - break; - case '\n': - if( escape ) *ptr++ = '\\'; - *ptr++ = '\\'; - *ptr++ = 'n'; - break; - case '\r': - if( escape ) *ptr++ = '\\'; - *ptr++ = '\\'; - *ptr++ = 'r'; - break; - case '\t': - if( escape ) *ptr++ = '\\'; - *ptr++ = '\\'; - *ptr++ = 't'; - break; - default: - *ptr++ = *in; - break; - } - in++; - } - return ptr; -} - -/* escape means we return '\N' for copy mode, otherwise we return just NULL */ -char *pgsql_store_tags(struct keyval *tags, int escape) -{ - static char *buffer; - static int buflen; - - char *ptr; - struct keyval *i; - int first; - - int countlist = countList(tags); - if( countlist == 0 ) - { - if( escape ) - return "\\N"; - else - return NULL; - } - - if( buflen <= countlist * 24 ) /* LE so 0 always matches */ - { - buflen = ((countlist * 24) | 4095) + 1; /* Round up to next page */ - buffer = realloc( buffer, buflen ); - } -_restart: - - ptr = buffer; - first = 1; - *ptr++ = '{'; - /* The lists are circular, exit when we reach the head again */ - for( i=tags->next; i->key; i = i->next ) - { - int maxlen = (strlen(i->key) + strlen(i->value)) * 4; - if( (ptr+maxlen-buffer) > (buflen-20) ) /* Almost overflowed? */ - { - buflen <<= 1; - buffer = realloc( buffer, buflen ); - - goto _restart; - } - if( !first ) *ptr++ = ','; - *ptr++ = '"'; - ptr = escape_tag( ptr, i->key, escape ); - *ptr++ = '"'; - *ptr++ = ','; - *ptr++ = '"'; - ptr = escape_tag( ptr, i->value, escape ); - *ptr++ = '"'; - - first=0; - } - - *ptr++ = '}'; - *ptr++ = 0; - - return buffer; -} - -/* Decodes a portion of an array literal from postgres */ -/* Argument should point to beginning of literal, on return points to delimiter */ -static const char *decode_upto( const char *src, char *dst ) -{ - int quoted = (*src == '"'); - if( quoted ) src++; - - while( quoted ? (*src != '"') : (*src != ',' && *src != '}') ) - { - if( *src == '\\' ) - { - switch( src[1] ) - { - case 'n': *dst++ = '\n'; break; - case 't': *dst++ = '\t'; break; - default: *dst++ = src[1]; break; - } - src+=2; - } - else - *dst++ = *src++; - } - if( quoted ) src++; - *dst = 0; - return src; -} - -static void pgsql_parse_tags( const char *string, struct keyval *tags ) -{ - char key[1024]; - char val[1024]; - - if( *string == '\0' ) - return; - - if( *string++ != '{' ) - return; - while( *string != '}' ) - { - string = decode_upto( string, key ); - /* String points to the comma */ - string++; - string = decode_upto( string, val ); - /* String points to the comma or closing '}' */ - addItem( tags, key, val, 0 ); - if( *string == ',' ) - string++; - } -} - -/* Parses an array of integers */ -static void pgsql_parse_nodes(const char *src, osmid_t *nds, int nd_count ) -{ - int count = 0; - const char *string = src; - - if( *string++ != '{' ) - return; - while( *string != '}' ) - { - char *ptr; - nds[count] = strtoosmid( string, &ptr, 10 ); - string = ptr; - if( *string == ',' ) - string++; - count++; - } - if( count != nd_count ) - { - fprintf( stderr, "parse_nodes problem: '%s' expected %d got %d\n", src, nd_count, count ); - exit_nicely(); - } -} - -static int pgsql_endCopy( struct table_desc *table) -{ - PGresult *res; - PGconn *sql_conn; - int stop; - /* Terminate any pending COPY */ - if (table->copyMode) { - sql_conn = table->sql_conn; - stop = PQputCopyEnd(sql_conn, NULL); - if (stop != 1) { - fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn)); - exit_nicely(); - } - - res = PQgetResult(sql_conn); - if (PQresultStatus(res) != PGRES_COMMAND_OK) { - fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn)); - PQclear(res); - exit_nicely(); - } - PQclear(res); - table->copyMode = 0; - } - return 0; -} - -static int pgsql_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags) -{ - /* Four params: id, lat, lon, tags */ - char *paramValues[4]; - char *buffer; - - if( node_table->copyMode ) - { - char *tag_buf = pgsql_store_tags(tags,1); - int length = strlen(tag_buf) + 64; - buffer = alloca( length ); -#ifdef FIXED_POINT - if( snprintf( buffer, length, "%" PRIdOSMID "\t%d\t%d\t%s\n", id, DOUBLE_TO_FIX(lat), DOUBLE_TO_FIX(lon), tag_buf ) > (length-10) ) - { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; } -#else - if( snprintf( buffer, length, "%" PRIdOSMID "\t%.10f\t%.10f\t%s\n", id, lat, lon, tag_buf ) > (length-10) ) - { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; } -#endif - return pgsql_CopyData(__FUNCTION__, node_table->sql_conn, buffer); - } - buffer = alloca(64); - paramValues[0] = buffer; - paramValues[1] = paramValues[0] + sprintf( paramValues[0], "%" PRIdOSMID, id ) + 1; -#ifdef FIXED_POINT - paramValues[2] = paramValues[1] + sprintf( paramValues[1], "%d", DOUBLE_TO_FIX(lat) ) + 1; - sprintf( paramValues[2], "%d", DOUBLE_TO_FIX(lon) ); -#else - paramValues[2] = paramValues[1] + sprintf( paramValues[1], "%.10f", lat ) + 1; - sprintf( paramValues[2], "%.10f", lon ); -#endif - paramValues[3] = pgsql_store_tags(tags,0); - pgsql_execPrepared(node_table->sql_conn, "insert_node", 4, (const char * const *)paramValues, PGRES_COMMAND_OK); - return 0; -} - -static int middle_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags) { - ram_cache_nodes_set( id, lat, lon, tags ); - - return (out_options->flat_node_cache_enabled) ? persistent_cache_nodes_set(id, lat, lon) : pgsql_nodes_set(id, lat, lon, tags); -} - - -static int pgsql_nodes_get(struct osmNode *out, osmid_t id) -{ - PGresult *res; - char tmp[16]; - char const *paramValues[1]; - PGconn *sql_conn = node_table->sql_conn; - - /* Make sure we're out of copy mode */ - pgsql_endCopy( node_table ); - - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); - paramValues[0] = tmp; - - res = pgsql_execPrepared(sql_conn, "get_node", 1, paramValues, PGRES_TUPLES_OK); - - if (PQntuples(res) != 1) { - PQclear(res); - return 1; - } - -#ifdef FIXED_POINT - out->lat = FIX_TO_DOUBLE(strtol(PQgetvalue(res, 0, 0), NULL, 10)); - out->lon = FIX_TO_DOUBLE(strtol(PQgetvalue(res, 0, 1), NULL, 10)); -#else - out->lat = strtod(PQgetvalue(res, 0, 0), NULL); - out->lon = strtod(PQgetvalue(res, 0, 1), NULL); -#endif - PQclear(res); - return 0; -} - -/* Currently not used -static int middle_nodes_get(struct osmNode *out, osmid_t id) -{ - / * Check cache first * / - if( ram_cache_nodes_get( out, id ) == 0 ) - return 0; - - return (out_options->flat_node_cache_enabled) ? persistent_cache_nodes_get(out, id) : pgsql_nodes_get(out, id); -}*/ - - -/* This should be made more efficient by using an IN(ARRAY[]) construct */ -static int pgsql_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, int nd_count) -{ - char tmp[16]; - char *tmp2; - int count, countDB, countPG, i,j; - osmid_t *ndidspg; - struct osmNode *nodespg; - char const *paramValues[1]; - - PGresult *res; - PGconn *sql_conn = node_table->sql_conn; - - count = 0; countDB = 0; - - tmp2 = malloc(sizeof(char)*nd_count*16); - if (tmp2 == NULL) return 0; /*failed to allocate memory, return */ - - /* create a list of ids in tmp2 to query the database */ - sprintf(tmp2, "{"); - for( i=0; iflat_node_cache_enabled) ? persistent_cache_nodes_get_list(nodes, ndids, nd_count) : pgsql_nodes_get_list(nodes, ndids, nd_count); -} - -static int pgsql_nodes_delete(osmid_t osm_id) -{ - char const *paramValues[1]; - char buffer[64]; - /* Make sure we're out of copy mode */ - pgsql_endCopy( node_table ); - - sprintf( buffer, "%" PRIdOSMID, osm_id ); - paramValues[0] = buffer; - pgsql_execPrepared(node_table->sql_conn, "delete_node", 1, paramValues, PGRES_COMMAND_OK ); - return 0; -} - -static int middle_nodes_delete(osmid_t osm_id) -{ - return ((out_options->flat_node_cache_enabled) ? persistent_cache_nodes_set(osm_id, NAN, NAN) : pgsql_nodes_delete(osm_id)); -} - -static int pgsql_node_changed(osmid_t osm_id) -{ - char const *paramValues[1]; - char buffer[64]; - /* Make sure we're out of copy mode */ - pgsql_endCopy( way_table ); - pgsql_endCopy( rel_table ); - - sprintf( buffer, "%" PRIdOSMID, osm_id ); - paramValues[0] = buffer; - pgsql_execPrepared(way_table->sql_conn, "node_changed_mark", 1, paramValues, PGRES_COMMAND_OK ); - pgsql_execPrepared(rel_table->sql_conn, "node_changed_mark", 1, paramValues, PGRES_COMMAND_OK ); - return 0; -} - -static int pgsql_ways_set(osmid_t way_id, osmid_t *nds, int nd_count, struct keyval *tags, int pending) -{ - /* Three params: id, nodes, tags, pending */ - char *paramValues[4]; - char *buffer; - - if( way_table->copyMode ) - { - char *tag_buf = pgsql_store_tags(tags,1); - char *node_buf = pgsql_store_nodes(nds, nd_count); - int length = strlen(tag_buf) + strlen(node_buf) + 64; - buffer = alloca(length); - if( snprintf( buffer, length, "%" PRIdOSMID "\t%s\t%s\t%c\n", - way_id, node_buf, tag_buf, pending?'t':'f' ) > (length-10) ) - { fprintf( stderr, "buffer overflow way id %" PRIdOSMID "\n", way_id); return 1; } - return pgsql_CopyData(__FUNCTION__, way_table->sql_conn, buffer); - } - buffer = alloca(64); - paramValues[0] = buffer; - paramValues[3] = paramValues[0] + sprintf( paramValues[0], "%" PRIdOSMID, way_id ) + 1; - sprintf( paramValues[3], "%c", pending?'t':'f' ); - paramValues[1] = pgsql_store_nodes(nds, nd_count); - paramValues[2] = pgsql_store_tags(tags,0); - pgsql_execPrepared(way_table->sql_conn, "insert_way", 4, (const char * const *)paramValues, PGRES_COMMAND_OK); - return 0; -} - -/* Caller is responsible for freeing nodesptr & resetList(tags) */ -static int pgsql_ways_get(osmid_t id, struct keyval *tags, struct osmNode **nodes_ptr, int *count_ptr) -{ - PGresult *res; - char tmp[16]; - char const *paramValues[1]; - PGconn *sql_conn = way_table->sql_conn; - int num_nodes; - osmid_t *list; - - /* Make sure we're out of copy mode */ - pgsql_endCopy( way_table ); - - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); - paramValues[0] = tmp; - - res = pgsql_execPrepared(sql_conn, "get_way", 1, paramValues, PGRES_TUPLES_OK); - - if (PQntuples(res) != 1) { - PQclear(res); - return 1; - } - - pgsql_parse_tags( PQgetvalue(res, 0, 1), tags ); - - num_nodes = strtol(PQgetvalue(res, 0, 2), NULL, 10); - list = alloca(sizeof(osmid_t)*num_nodes ); - *nodes_ptr = malloc(sizeof(struct osmNode) * num_nodes); - pgsql_parse_nodes( PQgetvalue(res, 0, 0), list, num_nodes); - - *count_ptr = out_options->flat_node_cache_enabled ? - persistent_cache_nodes_get_list(*nodes_ptr, list, num_nodes) : - pgsql_nodes_get_list( *nodes_ptr, list, num_nodes); - PQclear(res); - return 0; -} - -static int pgsql_ways_get_list(osmid_t *ids, int way_count, osmid_t **way_ids, struct keyval *tags, struct osmNode **nodes_ptr, int *count_ptr) { - - char tmp[16]; - char *tmp2; - int count, countPG, i, j; - osmid_t *wayidspg; - char const *paramValues[1]; - int num_nodes; - osmid_t *list; - - PGresult *res; - PGconn *sql_conn = way_table->sql_conn; - - *way_ids = malloc( sizeof(osmid_t) * (way_count + 1)); - if (way_count == 0) return 0; - - tmp2 = malloc(sizeof(char)*way_count*16); - if (tmp2 == NULL) return 0; /*failed to allocate memory, return */ - - /* create a list of ids in tmp2 to query the database */ - sprintf(tmp2, "{"); - for( i=0; iflat_node_cache_enabled ? - persistent_cache_nodes_get_list(nodes_ptr[count], list, num_nodes) : - pgsql_nodes_get_list( nodes_ptr[count], list, num_nodes); - - count++; - initList(&(tags[count])); - } - } - } - - PQclear(res); - free(tmp2); - free(wayidspg); - - return count; -} - -static int pgsql_ways_done(osmid_t id) -{ - char tmp[16]; - char const *paramValues[1]; - PGconn *sql_conn = way_table->sql_conn; - - /* Make sure we're out of copy mode */ - pgsql_endCopy( way_table ); - - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); - paramValues[0] = tmp; - - pgsql_execPrepared(sql_conn, "way_done", 1, paramValues, PGRES_COMMAND_OK); - - return 0; -} - -static int pgsql_ways_delete(osmid_t osm_id) -{ - char const *paramValues[1]; - char buffer[64]; - /* Make sure we're out of copy mode */ - pgsql_endCopy( way_table ); - - sprintf( buffer, "%" PRIdOSMID, osm_id ); - paramValues[0] = buffer; - pgsql_execPrepared(way_table->sql_conn, "delete_way", 1, paramValues, PGRES_COMMAND_OK ); - return 0; -} - -static void pgsql_iterate_ways(int (*callback)(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists)) -{ - int noProcs = out_options->num_procs; - int pid = 0; - PGresult *res_ways; - int i, p, count = 0; - /* The flag we pass to indicate that the way in question might exist already in the database */ - int exists = Append; - - time_t start, end; - time(&start); -#if HAVE_MMAP - struct progress_info *info = 0; - if(noProcs > 1) { - info = mmap(0, sizeof(struct progress_info)*noProcs, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); - info[0].finished = HELPER_STATE_CONNECTED; - for (i = 1; i < noProcs; i++) { - info[i].finished = HELPER_STATE_UNINITIALIZED; /* Register that the process was not yet initialised; */ - } - } -#endif - fprintf(stderr, "\nGoing over pending ways...\n"); - - /* Make sure we're out of copy mode */ - pgsql_endCopy( way_table ); - - if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache(); - - res_ways = pgsql_execPrepared(way_table->sql_conn, "pending_ways", 0, NULL, PGRES_TUPLES_OK); - - fprintf(stderr, "\t%i ways are pending\n", PQntuples(res_ways)); - - - /** - * To speed up processing of pending ways, fork noProcs worker processes - * each of which independently goes through an equal subset of the pending ways array - */ - fprintf(stderr, "\nUsing %i helper-processes\n", noProcs); -#ifdef HAVE_FORK - for (p = 1; p < noProcs; p++) { - pid=fork(); - if (pid==0) { -#if HAVE_MMAP - info[p].finished = HELPER_STATE_FORKED; -#endif - break; - } - if (pid==-1) { -#if HAVE_MMAP - info[p].finished = HELPER_STATE_FAILED; - fprintf(stderr,"WARNING: Failed to fork helper processes %i. Trying to recover.\n", p); -#else - fprintf(stderr,"ERROR: Failed to fork helper processes. Can't recover! \n"); - exit_nicely(); -#endif - } - } -#endif - if ((pid == 0) && (noProcs > 1)) { - /* After forking, need to reconnect to the postgresql db */ - if ((pgsql_connect(out_options) != 0) || (out_options->out->connect(out_options, 1) != 0)) { -#if HAVE_MMAP - info[p].finished = HELPER_STATE_FAILED; -#else - fprintf(stderr,"\n\n!!!FATAL: Helper process failed, but can't compensate. Your DB will be broken and corrupt!!!!\n\n"); -#endif - exit_nicely(); - }; - } else { - p = 0; - } - - if (out_options->flat_node_cache_enabled) init_node_persistent_cache(out_options,1); /* at this point we always want to be in append mode, to not delete and recreate the node cache file */ - - /* Only start an extended transaction on the ways table, - * which should cover the bulk of the update statements. - * The nodes table should not be written to in this phase. - * The relations table can't be wrapped in an extended - * transaction, as with prallel processing it may deadlock. - * Updating a way will trigger an update of the pending status - * on connected relations. This should not be as many updates, - * so in combination with the synchronous_comit = off it should be fine. - * - */ - if (tables[t_way].start) { - pgsql_endCopy(&tables[t_way]); - pgsql_exec(tables[t_way].sql_conn, PGRES_COMMAND_OK, "%s", tables[t_way].start); - tables[t_way].transactionMode = 1; - } - -#if HAVE_MMAP - if (noProcs > 1) { - info[p].finished = HELPER_STATE_CONNECTED; - /* Syncronize all processes to make sure they have all run through the initialisation steps */ - int all_processes_initialised = 0; - while (all_processes_initialised == 0) { - all_processes_initialised = 1; - for (i = 0; i < noProcs; i++) { - if (info[i].finished < 0) { - all_processes_initialised = 0; - sleep(1); - } - } - } - - /* As we process the pending ways in steps of noProcs, - we need to make sure that all processes correctly forked - and have connected to the db. Otherwise we need to readjust - the step size of going through the pending ways array */ - int noProcsTmp = noProcs; - int pTmp = p; - for (i = 0; i < noProcs; i++) { - if (info[i].finished == HELPER_STATE_FAILED) { - noProcsTmp--; - if (i < p) pTmp--; - } - } - info[p].finished = HELPER_STATE_RUNNING; - p = pTmp; /* reset the process number to account for failed processes */ - - /* As we have potentially changed the process number assignment, - we need to synchronize on all processes having performed the reassignment - as otherwise multiple process might have the same number and overwrite - the info fields incorrectly. - */ - all_processes_initialised = 0; - while (all_processes_initialised == 0) { - all_processes_initialised = 1; - for (i = 0; i < noProcs; i++) { - if (info[i].finished == HELPER_STATE_CONNECTED) { - /* Process is connected, but hasn't performed the re-assignment of p. */ - all_processes_initialised = 0; - sleep(1); - break; - } - } - } - noProcs = noProcsTmp; - } -#endif - - /* some spaces at end, so that processings outputs get cleaned if already existing */ - fprintf(stderr, "\rHelper process %i out of %i initialised \n", p, noProcs); - /* Use a stride length of the number of worker processes, - starting with an offset for each worker process p */ - for (i = p; i < PQntuples(res_ways); i+= noProcs) { - osmid_t id = strtoosmid(PQgetvalue(res_ways, i, 0), NULL, 10); - struct keyval tags; - struct osmNode *nodes; - int nd_count; - - if (count++ %1000 == 0) { - time(&end); -#if HAVE_MMAP - if(info) - { - double rate = 0; - int n, total = 0, finished = 0; - struct progress_info f; - - f.start = start; - f.end = end; - f.count = count; - f.finished = HELPER_STATE_RUNNING; - info[p] = f; - for(n = 0; n < noProcs; ++n) - { - f = info[n]; - total += f.count; - finished += f.finished; - if(f.end > f.start) - rate += (double)f.count / (double)(f.end - f.start); - } - fprintf(stderr, "\rprocessing way (%dk) at %.2fk/s (done %d of %d)", total/1000, rate/1000.0, finished, noProcs); - } - else -#endif - { - fprintf(stderr, "\rprocessing way (%dk) at %.2fk/s", count/1000, - end > start ? ((double)count / 1000.0 / (double)(end - start)) : 0); - } - } - - initList(&tags); - if( pgsql_ways_get(id, &tags, &nodes, &nd_count) ) - continue; - - callback(id, &tags, nodes, nd_count, exists); - pgsql_ways_done( id ); - - free(nodes); - resetList(&tags); - } - - if (tables[t_way].stop && tables[t_way].transactionMode) { - pgsql_exec(tables[t_way].sql_conn, PGRES_COMMAND_OK, "%s", tables[t_way].stop); - tables[t_way].transactionMode = 0; - } - - time(&end); -#if HAVE_MMAP - if(info) - { - struct progress_info f; - f.start = start; - f.end = end; - f.count = count; - f.finished = 1; - info[p] = f; - } -#endif - fprintf(stderr, "\rProcess %i finished processing %i ways in %i sec\n", p, count, (int)(end - start)); - - if ((pid == 0) && (noProcs > 1)) { - pgsql_cleanup(); - out_options->out->close(1); - if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache(); - exit(0); - } else { - for (p = 0; p < noProcs; p++) wait(NULL); - fprintf(stderr, "\nAll child processes exited\n"); - } - -#if HAVE_MMAP - munmap(info, sizeof(struct progress_info)*noProcs); -#endif - - fprintf(stderr, "\n"); - time(&end); - if (end - start > 0) - fprintf(stderr, "%i Pending ways took %ds at a rate of %.2f/s\n",PQntuples(res_ways), (int)(end - start), - ((double)PQntuples(res_ways) / (double)(end - start))); - PQclear(res_ways); -} - -static int pgsql_way_changed(osmid_t osm_id) -{ - char const *paramValues[1]; - char buffer[64]; - /* Make sure we're out of copy mode */ - pgsql_endCopy( rel_table ); - - sprintf( buffer, "%" PRIdOSMID, osm_id ); - paramValues[0] = buffer; - pgsql_execPrepared(rel_table->sql_conn, "way_changed_mark", 1, paramValues, PGRES_COMMAND_OK ); - return 0; -} - -static int pgsql_rels_set(osmid_t id, struct member *members, int member_count, struct keyval *tags) -{ - /* Params: id, way_off, rel_off, parts, members, tags */ - char *paramValues[6]; - char *buffer; - int i; - struct keyval member_list; - char buf[64]; - - osmid_t node_parts[member_count], - way_parts[member_count], - rel_parts[member_count]; - int node_count = 0, way_count = 0, rel_count = 0; - - osmid_t all_parts[member_count]; - int all_count = 0; - initList( &member_list ); - for( i=0; icopyMode ) - { - char *tag_buf = strdup(pgsql_store_tags(tags,1)); - char *member_buf = pgsql_store_tags(&member_list,1); - char *parts_buf = pgsql_store_nodes(all_parts, all_count); - int length = strlen(member_buf) + strlen(tag_buf) + strlen(parts_buf) + 64; - buffer = alloca(length); - if( snprintf( buffer, length, "%" PRIdOSMID "\t%d\t%d\t%s\t%s\t%s\tf\n", - id, node_count, node_count+way_count, parts_buf, member_buf, tag_buf ) > (length-10) ) - { fprintf( stderr, "buffer overflow relation id %" PRIdOSMID "\n", id); return 1; } - free(tag_buf); - resetList(&member_list); - return pgsql_CopyData(__FUNCTION__, rel_table->sql_conn, buffer); - } - buffer = alloca(64); - paramValues[0] = buffer; - paramValues[1] = paramValues[0] + sprintf( paramValues[0], "%" PRIdOSMID, id ) + 1; - paramValues[2] = paramValues[1] + sprintf( paramValues[1], "%d", node_count ) + 1; - sprintf( paramValues[2], "%d", node_count+way_count ); - paramValues[3] = pgsql_store_nodes(all_parts, all_count); - paramValues[4] = pgsql_store_tags(&member_list,0); - if( paramValues[4] ) - paramValues[4] = strdup(paramValues[4]); - paramValues[5] = pgsql_store_tags(tags,0); - pgsql_execPrepared(rel_table->sql_conn, "insert_rel", 6, (const char * const *)paramValues, PGRES_COMMAND_OK); - if( paramValues[4] ) - free(paramValues[4]); - resetList(&member_list); - return 0; -} - -/* Caller is responsible for freeing members & resetList(tags) */ -static int pgsql_rels_get(osmid_t id, struct member **members, int *member_count, struct keyval *tags) -{ - PGresult *res; - char tmp[16]; - char const *paramValues[1]; - PGconn *sql_conn = rel_table->sql_conn; - struct keyval member_temp; - char tag; - int num_members; - struct member *list; - int i=0; - struct keyval *item; - - /* Make sure we're out of copy mode */ - pgsql_endCopy( rel_table ); - - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); - paramValues[0] = tmp; - - res = pgsql_execPrepared(sql_conn, "get_rel", 1, paramValues, PGRES_TUPLES_OK); - /* Fields are: members, tags, member_count */ - - if (PQntuples(res) != 1) { - PQclear(res); - return 1; - } - - pgsql_parse_tags( PQgetvalue(res, 0, 1), tags ); - initList(&member_temp); - pgsql_parse_tags( PQgetvalue(res, 0, 0), &member_temp ); - - num_members = strtol(PQgetvalue(res, 0, 2), NULL, 10); - list = malloc( sizeof(struct member)*num_members ); - - while( (item = popItem(&member_temp)) ) - { - if( i >= num_members ) - { - fprintf(stderr, "Unexpected member_count reading relation %" PRIdOSMID "\n", id); - exit_nicely(); - } - tag = item->key[0]; - list[i].type = (tag == 'n')?OSMTYPE_NODE:(tag == 'w')?OSMTYPE_WAY:(tag == 'r')?OSMTYPE_RELATION:-1; - list[i].id = strtoosmid(item->key+1, NULL, 10 ); - list[i].role = strdup( item->value ); - freeItem(item); - i++; - } - *members = list; - *member_count = num_members; - PQclear(res); - return 0; -} - -static int pgsql_rels_done(osmid_t id) -{ - char tmp[16]; - char const *paramValues[1]; - PGconn *sql_conn = rel_table->sql_conn; - - /* Make sure we're out of copy mode */ - pgsql_endCopy( rel_table ); - - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); - paramValues[0] = tmp; - - pgsql_execPrepared(sql_conn, "rel_done", 1, paramValues, PGRES_COMMAND_OK); - - return 0; -} - -static int pgsql_rels_delete(osmid_t osm_id) -{ - char const *paramValues[1]; - char buffer[64]; - /* Make sure we're out of copy mode */ - pgsql_endCopy( rel_table ); - - sprintf( buffer, "%" PRIdOSMID, osm_id ); - paramValues[0] = buffer; - pgsql_execPrepared(rel_table->sql_conn, "delete_rel", 1, paramValues, PGRES_COMMAND_OK ); - return 0; -} - -static void pgsql_iterate_relations(int (*callback)(osmid_t id, struct member *members, int member_count, struct keyval *tags, int exists)) -{ - PGresult *res_rels; - int noProcs = out_options->num_procs; - int pid; - int i, p, count = 0; - /* The flag we pass to indicate that the way in question might exist already in the database */ - int exists = Append; - - time_t start, end; - time(&start); -#if HAVE_MMAP - struct progress_info *info = 0; - if(noProcs > 1) { - info = mmap(0, sizeof(struct progress_info)*noProcs, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); - info[0].finished = HELPER_STATE_CONNECTED; - for (i = 1; i < noProcs; i++) { - info[i].finished = HELPER_STATE_UNINITIALIZED; /* Register that the process was not yet initialised; */ - } - } -#endif - fprintf(stderr, "\nGoing over pending relations...\n"); - - /* Make sure we're out of copy mode */ - pgsql_endCopy( rel_table ); - - if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache(); - - res_rels = pgsql_execPrepared(rel_table->sql_conn, "pending_rels", 0, NULL, PGRES_TUPLES_OK); - - fprintf(stderr, "\t%i relations are pending\n", PQntuples(res_rels)); - - fprintf(stderr, "\nUsing %i helper-processes\n", noProcs); - pid = 0; -#ifdef HAVE_FORK - for (p = 1; p < noProcs; p++) { - pid=fork(); - if (pid==0) { -#if HAVE_MMAP - info[p].finished = HELPER_STATE_FORKED; -#endif - break; - } - if (pid==-1) { -#if HAVE_MMAP - info[p].finished = HELPER_STATE_FAILED; - fprintf(stderr,"WARNING: Failed to fork helper processes %i. Trying to recover.\n", p); -#else - fprintf(stderr,"ERROR: Failed to fork helper processes. Can't recover! \n"); - exit_nicely(); -#endif - } - } -#endif - if ((pid == 0) && (noProcs > 1)) { - if ((out_options->out->connect(out_options, 0) != 0) || (pgsql_connect(out_options) != 0)) { -#if HAVE_MMAP - info[p].finished = HELPER_STATE_FAILED; -#endif - exit_nicely(); - }; - } else { - p = 0; - } - - if (out_options->flat_node_cache_enabled) init_node_persistent_cache(out_options, 1); /* at this point we always want to be in append mode, to not delete and recreate the node cache file */ - -#if HAVE_MMAP - if (noProcs > 1) { - info[p].finished = HELPER_STATE_CONNECTED; - /* Syncronize all processes to make sure they have all run through the initialisation steps */ - int all_processes_initialised = 0; - while (all_processes_initialised == 0) { - all_processes_initialised = 1; - for (i = 0; i < noProcs; i++) { - if (info[i].finished < 0) { - all_processes_initialised = 0; - sleep(1); - } - } - } - - /* As we process the pending ways in steps of noProcs, - we need to make sure that all processes correctly forked - and have connected to the db. Otherwise we need to readjust - the step size of going through the pending ways array */ - int noProcsTmp = noProcs; - int pTmp = p; - for (i = 0; i < noProcs; i++) { - if (info[i].finished == HELPER_STATE_FAILED) { - noProcsTmp--; - if (i < p) pTmp--; - } - } - info[p].finished = HELPER_STATE_RUNNING; - p = pTmp; /* reset the process number to account for failed processes */ - - /* As we have potentially changed the process number assignment, - we need to synchronize on all processes having performed the reassignment - as otherwise multiple process might have the same number and overwrite - the info fields incorrectly. - */ - all_processes_initialised = 0; - while (all_processes_initialised == 0) { - all_processes_initialised = 1; - for (i = 0; i < noProcs; i++) { - if (info[i].finished == HELPER_STATE_CONNECTED) { - /* Process is connected, but hasn't performed the re-assignment of p. */ - all_processes_initialised = 0; - sleep(1); - break; - } - } - } - noProcs = noProcsTmp; - } -#endif - - for (i = p; i < PQntuples(res_rels); i+= noProcs) { - osmid_t id = strtoosmid(PQgetvalue(res_rels, i, 0), NULL, 10); - struct keyval tags; - struct member *members; - int member_count; - - if (count++ %10 == 0) { - time(&end); -#if HAVE_MMAP - if(info) - { - double rate = 0; - int n, total = 0, finished = 0; - struct progress_info f; - - f.start = start; - f.end = end; - f.count = count; - f.finished = HELPER_STATE_RUNNING; - info[p] = f; - for(n = 0; n < noProcs; ++n) - { - f = info[n]; - total += f.count; - finished += f.finished; - if(f.end > f.start) - rate += (double)f.count / (double)(f.end - f.start); - } - fprintf(stderr, "\rprocessing relation (%d) at %.2f/s (done %d of %d)", total, rate, finished, noProcs); - } - else -#endif - { - fprintf(stderr, "\rprocessing relation (%d) at %.2f/s", count, - end > start ? ((double)count / (double)(end - start)) : 0); - } - } - - initList(&tags); - if( pgsql_rels_get(id, &members, &member_count, &tags) ) - continue; - - callback(id, members, member_count, &tags, exists); - pgsql_rels_done( id ); - - free(members); - resetList(&tags); - } - time(&end); -#if HAVE_MMAP - if(info) - { - struct progress_info f; - f.start = start; - f.end = end; - f.count = count; - f.finished = 1; - info[p] = f; - } -#endif - fprintf(stderr, "\rProcess %i finished processing %i relations in %i sec\n", p, count, (int)(end - start)); - - if ((pid == 0) && (noProcs > 1)) { - pgsql_cleanup(); - out_options->out->close(0); - if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache(); - exit(0); - } else { - for (p = 0; p < noProcs; p++) wait(NULL); - fprintf(stderr, "\nAll child processes exited\n"); - } - -#if HAVE_MMAP - munmap(info, sizeof(struct progress_info)*noProcs); -#endif - time(&end); - if (end - start > 0) - fprintf(stderr, "%i Pending relations took %ds at a rate of %.2f/s\n",PQntuples(res_rels), (int)(end - start), ((double)PQntuples(res_rels) / (double)(end - start))); - PQclear(res_rels); - fprintf(stderr, "\n"); - -} - -static int pgsql_rel_changed(osmid_t osm_id) -{ - char const *paramValues[1]; - char buffer[64]; - /* Make sure we're out of copy mode */ - pgsql_endCopy( rel_table ); - - sprintf( buffer, "%" PRIdOSMID, osm_id ); - paramValues[0] = buffer; - pgsql_execPrepared(rel_table->sql_conn, "rel_changed_mark", 1, paramValues, PGRES_COMMAND_OK ); - return 0; -} - -static void pgsql_analyze(void) -{ - int i; - - for (i=0; iprefix) { - strcpy(dest, options->prefix); - dest += strlen(options->prefix); - copied = 1; - } - source+=2; - continue; - } else if (*(source+1) == 't') { - if (options->tblsslim_data) { - strcpy(dest, options->tblsslim_data); - dest += strlen(options->tblsslim_data); - copied = 1; - } - source+=2; - continue; - } else if (*(source+1) == 'i') { - if (options->tblsslim_index) { - strcpy(dest, options->tblsslim_index); - dest += strlen(options->tblsslim_index); - copied = 1; - } - source+=2; - continue; - } else if (*(source+1) == 'm') { - if (options->unlogged) { - strcpy(dest, "UNLOGGED"); - dest += 8; - copied = 1; - } - source+=2; - continue; - } - } - *(dest++) = *(source++); - } - *dest = 0; - *string = strdup(buffer); -} - -static int build_indexes; - - -static int pgsql_start(const struct output_options *options) -{ - PGresult *res; - int i; - int dropcreate = !options->append; - char * sql; - - scale = options->scale; - Append = options->append; - - out_options = options; - - init_node_ram_cache( options->alloc_chunkwise | ALLOC_LOSSY, options->cache, scale); - if (options->flat_node_cache_enabled) init_node_persistent_cache(options, options->append); - - fprintf(stderr, "Mid: pgsql, scale=%d cache=%d\n", scale, options->cache); - - /* We use a connection per table to enable the use of COPY */ - for (i=0; iconninfo); - - /* Check to see that the backend connection was successfully made */ - if (PQstatus(sql_conn) != CONNECTION_OK) { - fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn)); - exit_nicely(); - } - tables[i].sql_conn = sql_conn; - - /* - * To allow for parallelisation, the second phase (iterate_ways), cannot be run - * in an extended transaction and each update statement is its own transaction. - * Therefore commit rate of postgresql is very important to ensure high speed. - * If fsync is enabled to ensure safe transactions, the commit rate can be very low. - * To compensate for this, one can set the postgresql parameter synchronous_commit - * to off. This means an update statement returns to the client as success before the - * transaction is saved to disk via fsync, which in return allows to bunch up multiple - * transactions into a single fsync. This may result in some data loss in the case of a - * database crash. However, as we don't currently have the ability to restart a full osm2pgsql - * import session anyway, this is fine. Diff imports are also not effected, as the next - * diff import would simply deal with all pending ways that were not previously finished. - * This parameter does not effect safety from data corruption on the back-end. - */ - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;"); - - /* Not really the right place for this test, but we need a live - * connection that not used for anything else yet, and we'd like to - * warn users *before* we start doing mountains of work */ - if (i == t_node) - { - res = PQexec(sql_conn, "select 1 from pg_opclass where opcname='gist__intbig_ops'" ); - if(PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1) - { - /* intarray is problematic now; causes at least postgres 8.4 - * to not use the index on nodes[]/parts[] which slows diff - * updates to a crawl! - * If someone find a way to fix this rather than bow out here, - * please do.*/ - - fprintf(stderr, - "\n" - "The target database has the intarray contrib module loaded.\n" - "While required for earlier versions of osm2pgsql, intarray \n" - "is now unnecessary and will interfere with osm2pgsql's array\n" - "handling. Please use a database without intarray.\n\n"); - exit_nicely(); - } - PQclear(res); - - if (options->append) - { - sql = malloc (2048); - snprintf(sql, 2047, "SELECT id FROM %s LIMIT 1", tables[t_node].name); - res = PQexec(sql_conn, sql ); - free(sql); - sql = NULL; - if(PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1) - { - int size = PQfsize(res, 0); - if (size != sizeof(osmid_t)) - { - fprintf(stderr, - "\n" - "The target database has been created with %dbit ID fields,\n" - "but this version of osm2pgsql has been compiled to use %ldbit IDs.\n" - "You cannot append data to this database with this program.\n" - "Either re-create the database or use a matching osm2pgsql.\n\n", - size * 8, sizeof(osmid_t) * 8); - exit_nicely(); - } - } - PQclear(res); - } - - if(!options->append) - build_indexes = 1; - } - if (dropcreate) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s", tables[i].name); - } - - if (tables[i].start) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].start); - tables[i].transactionMode = 1; - } - - if (dropcreate && tables[i].create) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create); - if (tables[i].create_index) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create_index); - } - } - - - if (tables[i].prepare) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare); - } - - if (tables[i].prepare_intarray) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray); - } - - if (tables[i].copy) { - pgsql_exec(sql_conn, PGRES_COPY_IN, "%s", tables[i].copy); - tables[i].copyMode = 1; - } - } - - return 0; -} - -static void pgsql_commit(void) { - int i; - for (i=0; isql_conn; - - fprintf(stderr, "Stopping table: %s\n", table->name); - pgsql_endCopy(table); - time(&start); - if (!out_options->droptemp) - { - if (build_indexes && table->array_indexes) { - char *buffer = (char *) malloc(strlen(table->array_indexes) + 99); - /* we need to insert before the TABLESPACE setting, if any */ - char *insertpos = strstr(table->array_indexes, "TABLESPACE"); - if (!insertpos) insertpos = strchr(table->array_indexes, ';'); - - /* automatically insert FASTUPDATE=OFF when creating, - indexes for PostgreSQL 8.4 and higher - see http://lists.openstreetmap.org/pipermail/dev/2011-January/021704.html */ - if (insertpos && PQserverVersion(sql_conn) >= 80400) { - char old = *insertpos; - fprintf(stderr, "Building index on table: %s (fastupdate=off)\n", table->name); - *insertpos = 0; /* temporary null byte for following strcpy operation */ - strcpy(buffer, table->array_indexes); - *insertpos = old; /* restore old content */ - strcat(buffer, " WITH (FASTUPDATE=OFF)"); - strcat(buffer, insertpos); - } else { - fprintf(stderr, "Building index on table: %s\n", table->name); - strcpy(buffer, table->array_indexes); - } - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", buffer); - free(buffer); - } - } - else - { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "drop table %s", table->name); - } - PQfinish(sql_conn); - table->sql_conn = NULL; - time(&end); - fprintf(stderr, "Stopped table: %s in %is\n", table->name, (int)(end - start)); - return NULL; -} - -static void pgsql_stop(void) -{ - int i; -#ifdef HAVE_PTHREAD - pthread_t threads[num_tables]; -#endif - - free_node_ram_cache(); - if (out_options->flat_node_cache_enabled) shutdown_node_persistent_cache(); - -#ifdef HAVE_PTHREAD - for (i=0; i +#endif + +#ifdef HAVE_SYS_WAIT_H +#include +#endif + +#ifdef HAVE_MMAP +#include +#ifndef MAP_ANONYMOUS +#ifdef MAP_ANON +#define MAP_ANONYMOUS MAP_ANON +#endif +#endif +#endif + +#ifdef _WIN32 +using namespace std; +#endif + +#ifdef _MSC_VER +#define alloca _alloca +#endif + +#include + +#include "osmtypes.hpp" +#include "middle-pgsql.hpp" +#include "output-pgsql.hpp" +#include "options.hpp" +#include "node-ram-cache.hpp" +#include "node-persistent-cache.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum table_id { + t_node, t_way, t_rel +} ; + +middle_pgsql_t::table_desc::table_desc(const char *name_, + const char *start_, + const char *create_, + const char *create_index_, + const char *prepare_, + const char *prepare_intarray_, + const char *copy_, + const char *analyze_, + const char *stop_, + const char *array_indexes_) + : name(name_), + start(start_), + create(create_), + create_index(create_index_), + prepare(prepare_), + prepare_intarray(prepare_intarray_), + copy(copy_), + analyze(analyze_), + stop(stop_), + array_indexes(array_indexes_), + copyMode(0), + transactionMode(0), + sql_conn(NULL) +{} + +#define HELPER_STATE_UNINITIALIZED -1 +#define HELPER_STATE_FORKED -2 +#define HELPER_STATE_RUNNING 0 +#define HELPER_STATE_FINISHED 1 +#define HELPER_STATE_CONNECTED 2 +#define HELPER_STATE_FAILED 3 + +namespace { +char *pgsql_store_nodes(const idlist_t &nds) { + static char *buffer; + static size_t buflen; + + if( buflen <= nds.size() * 10 ) + { + buflen = ((nds.size() * 10) | 4095) + 1; // Round up to next page */ + buffer = (char *)realloc( buffer, buflen ); + } +_restart: + + char *ptr = buffer; + bool first = true; + *ptr++ = '{'; + for (idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) + { + if (!first) + *ptr++ = ','; + ptr += sprintf(ptr, "%" PRIdOSMID, *it); + + if( (size_t) (ptr-buffer) > (buflen-20) ) // Almost overflowed? */ + { + buflen <<= 1; + buffer = (char *)realloc( buffer, buflen ); + + goto _restart; + } + first = false; + } + + *ptr++ = '}'; + *ptr++ = 0; + + return buffer; +} + +// Special escape routine for escaping strings in array constants: double quote, backslash,newline, tab*/ +inline char *escape_tag( char *ptr, const std::string &in, bool escape ) +{ + BOOST_FOREACH(const char c, in) + { + switch(c) + { + case '"': + if( escape ) *ptr++ = '\\'; + *ptr++ = '\\'; + *ptr++ = '"'; + break; + case '\\': + if( escape ) *ptr++ = '\\'; + if( escape ) *ptr++ = '\\'; + *ptr++ = '\\'; + *ptr++ = '\\'; + break; + case '\n': + if( escape ) *ptr++ = '\\'; + *ptr++ = '\\'; + *ptr++ = 'n'; + break; + case '\r': + if( escape ) *ptr++ = '\\'; + *ptr++ = '\\'; + *ptr++ = 'r'; + break; + case '\t': + if( escape ) *ptr++ = '\\'; + *ptr++ = '\\'; + *ptr++ = 't'; + break; + default: + *ptr++ = c; + break; + } + } + return ptr; +} + +// escape means we return '\N' for copy mode, otherwise we return just NULL */ +const char *pgsql_store_tags(const taglist_t &tags, bool escape) +{ + static char *buffer; + static int buflen; + + int countlist = tags.size(); + if( countlist == 0 ) + { + if( escape ) + return "\\N"; + else + return NULL; + } + + if( buflen <= countlist * 24 ) // LE so 0 always matches */ + { + buflen = ((countlist * 24) | 4095) + 1; // Round up to next page */ + buffer = (char *)realloc( buffer, buflen ); + } +_restart: + + char *ptr = buffer; + bool first = true; + *ptr++ = '{'; + + for (taglist_t::const_iterator it = tags.begin(); it != tags.end(); ++it) + { + int maxlen = (it->key.length() + it->value.length()) * 4; + if( (ptr+maxlen-buffer) > (buflen-20) ) // Almost overflowed? */ + { + buflen <<= 1; + buffer = (char *)realloc( buffer, buflen ); + + goto _restart; + } + if( !first ) *ptr++ = ','; + *ptr++ = '"'; + ptr = escape_tag(ptr, it->key, escape); + *ptr++ = '"'; + *ptr++ = ','; + *ptr++ = '"'; + ptr = escape_tag(ptr, it->value, escape); + *ptr++ = '"'; + + first = false; + } + + *ptr++ = '}'; + *ptr++ = 0; + + return buffer; +} + +// Decodes a portion of an array literal from postgres */ +// Argument should point to beginning of literal, on return points to delimiter */ +inline const char *decode_upto( const char *src, char *dst ) +{ + int quoted = (*src == '"'); + if( quoted ) src++; + + while( quoted ? (*src != '"') : (*src != ',' && *src != '}') ) + { + if( *src == '\\' ) + { + switch( src[1] ) + { + case 'n': *dst++ = '\n'; break; + case 't': *dst++ = '\t'; break; + default: *dst++ = src[1]; break; + } + src+=2; + } + else + *dst++ = *src++; + } + if( quoted ) src++; + *dst = 0; + return src; +} + +void pgsql_parse_tags(const char *string, taglist_t &tags) +{ + char key[1024]; + char val[1024]; + + if( *string == '\0' ) + return; + + if( *string++ != '{' ) + return; + while( *string != '}' ) + { + string = decode_upto( string, key ); + // String points to the comma */ + string++; + string = decode_upto( string, val ); + // String points to the comma or closing '}' */ + tags.push_back(tag(key, val)); + if( *string == ',' ) + string++; + } +} + +// Parses an array of integers */ +void pgsql_parse_nodes(const char *string, idlist_t &nds) +{ + if( *string++ != '{' ) + return; + + while( *string != '}' ) + { + char *ptr; + nds.push_back(strtoosmid( string, &ptr, 10 )); + string = ptr; + if( *string == ',' ) + string++; + } +} + +int pgsql_endCopy(middle_pgsql_t::table_desc *table) +{ + // Terminate any pending COPY */ + if (table->copyMode) { + PGconn *sql_conn = table->sql_conn; + int stop = PQputCopyEnd(sql_conn, NULL); + if (stop != 1) { + fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn)); + util::exit_nicely(); + } + + PGresult *res = PQgetResult(sql_conn); + if (PQresultStatus(res) != PGRES_COMMAND_OK) { + fprintf(stderr, "COPY_END for %s failed: %s\n", table->copy, PQerrorMessage(sql_conn)); + PQclear(res); + util::exit_nicely(); + } + PQclear(res); + table->copyMode = 0; + } + return 0; +} +} // anonymous namespace + +int middle_pgsql_t::local_nodes_set(const osmid_t& id, const double& lat, + const double& lon, const taglist_t &tags) +{ + if( node_table->copyMode ) + { + const char *tag_buf = pgsql_store_tags(tags,1); + int length = strlen(tag_buf) + 64; + char *buffer = (char *)alloca( length ); +#ifdef FIXED_POINT + ramNode n(lon, lat); + if( snprintf(buffer, length, "%" PRIdOSMID "\t%d\t%d\t%s\n", + id, n.int_lat(), n.int_lon(), tag_buf ) > (length-10) ) + { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; } +#else + if( snprintf( buffer, length, "%" PRIdOSMID "\t%.10f\t%.10f\t%s\n", id, lat, lon, tag_buf ) > (length-10) ) + { fprintf( stderr, "buffer overflow node id %" PRIdOSMID "\n", id); return 1; } +#endif + pgsql_CopyData(__FUNCTION__, node_table->sql_conn, buffer); + return 0; + } + + // Four params: id, lat, lon, tags */ + const char *paramValues[4]; + char *buffer = (char *)alloca(64); + char *ptr = buffer; + paramValues[0] = ptr; + ptr += sprintf( ptr, "%" PRIdOSMID, id ) + 1; + paramValues[1] = ptr; +#ifdef FIXED_POINT + ramNode n(lon, lat); + ptr += sprintf(ptr, "%d", n.int_lat()) + 1; + paramValues[2] = ptr; + sprintf(ptr, "%d", n.int_lon()); +#else + ptr += sprintf( ptr, "%.10f", lat ) + 1; + paramValues[2] = ptr; + sprintf( ptr, "%.10f", lon ); +#endif + paramValues[3] = pgsql_store_tags(tags,0); + pgsql_execPrepared(node_table->sql_conn, "insert_node", 4, (const char * const *)paramValues, PGRES_COMMAND_OK); + return 0; +} + +// This should be made more efficient by using an IN(ARRAY[]) construct */ +int middle_pgsql_t::local_nodes_get_list(nodelist_t &out, const idlist_t nds) const +{ + assert(out.empty()); + + char tmp[16]; + + char *tmp2 = (char *)malloc(sizeof(char) * nds.size() * 16); + if (tmp2 == NULL) return 0; //failed to allocate memory, return */ + + + // create a list of ids in tmp2 to query the database */ + sprintf(tmp2, "{"); + int countDB = 0; + for(idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) { + // Check cache first */ + osmNode loc; + if (cache->get(&loc, *it) == 0) { + out.push_back(loc); + continue; + } + + countDB++; + // Mark nodes as needing to be fetched from the DB */ + out.push_back(osmNode()); + + snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it); + strncat(tmp2, tmp, sizeof(char)*(nds.size()*16 - 2)); + } + tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/ + + if (countDB == 0) { + free(tmp2); + return nds.size(); // All ids where in cache, so nothing more to do */ + } + + pgsql_endCopy(node_table); + + PGconn *sql_conn = node_table->sql_conn; + + char const *paramValues[1]; + paramValues[0] = tmp2; + PGresult *res = pgsql_execPrepared(sql_conn, "get_node_list", 1, paramValues, PGRES_TUPLES_OK); + int countPG = PQntuples(res); + + //store the pg results in a hashmap and telling it how many we expect + boost::unordered_map pg_nodes(countPG); + + for (int i = 0; i < countPG; i++) { + osmid_t id = strtoosmid(PQgetvalue(res, i, 0), NULL, 10); + osmNode node; +#ifdef FIXED_POINT + ramNode n((int) strtol(PQgetvalue(res, i, 2), NULL, 10), + (int) strtol(PQgetvalue(res, i, 1), NULL, 10)); + + node.lat = n.lat(); + node.lon = n.lon(); +#else + node.lat = strtod(PQgetvalue(res, i, 1), NULL); + node.lon = strtod(PQgetvalue(res, i, 2), NULL); +#endif + pg_nodes.emplace(id, node); + } + + PQclear(res); + free(tmp2); + + // If some of the nodes in the way don't exist, the returning list has holes. + // Merge the two lists removing any holes. + unsigned wrtidx = 0; + for (unsigned i = 0; i < nds.size(); ++i) { + if (std::isnan(out[i].lat)) { + boost::unordered_map::iterator found = pg_nodes.find(nds[i]); + if(found != pg_nodes.end()) { + out[wrtidx] = found->second; + ++wrtidx; + } + } else { + if (wrtidx < i) + out[wrtidx] = out[i]; + ++wrtidx; + } + } + out.resize(wrtidx); + + return wrtidx; +} + + +int middle_pgsql_t::nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags) { + cache->set( id, lat, lon, tags ); + + return (out_options->flat_node_cache_enabled) + ? persistent_cache->set(id, lat, lon) + : local_nodes_set(id, lat, lon, tags); +} + +int middle_pgsql_t::nodes_get_list(nodelist_t &out, const idlist_t nds) const +{ + return (out_options->flat_node_cache_enabled) + ? persistent_cache->get_list(out, nds) + : local_nodes_get_list(out, nds); +} + +int middle_pgsql_t::local_nodes_delete(osmid_t osm_id) +{ + char const *paramValues[1]; + char buffer[64]; + // Make sure we're out of copy mode */ + pgsql_endCopy( node_table ); + + sprintf( buffer, "%" PRIdOSMID, osm_id ); + paramValues[0] = buffer; + pgsql_execPrepared(node_table->sql_conn, "delete_node", 1, paramValues, PGRES_COMMAND_OK ); + return 0; +} + +int middle_pgsql_t::nodes_delete(osmid_t osm_id) +{ + return ((out_options->flat_node_cache_enabled) ? persistent_cache->set(osm_id, NAN, NAN) : local_nodes_delete(osm_id)); +} + +int middle_pgsql_t::node_changed(osmid_t osm_id) +{ + if (!mark_pending) + return 0; + + char const *paramValues[1]; + char buffer[64]; + // Make sure we're out of copy mode */ + pgsql_endCopy( way_table ); + pgsql_endCopy( rel_table ); + + sprintf( buffer, "%" PRIdOSMID, osm_id ); + paramValues[0] = buffer; + + //keep track of whatever ways and rels these nodes intersect + //TODO: dont need to stop the copy above since we are only reading? + PGresult* res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_node", 1, paramValues, PGRES_TUPLES_OK ); + for(int i = 0; i < PQntuples(res); ++i) + { + char *end; + osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10); + ways_pending_tracker->mark(marked); + } + PQclear(res); + + //do the rels too + res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels_by_node", 1, paramValues, PGRES_TUPLES_OK ); + for(int i = 0; i < PQntuples(res); ++i) + { + char *end; + osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10); + rels_pending_tracker->mark(marked); + } + PQclear(res); + + return 0; +} + +int middle_pgsql_t::ways_set(osmid_t way_id, const idlist_t &nds, const taglist_t &tags) +{ + // Three params: id, nodes, tags */ + const char *paramValues[4]; + char *buffer; + + if( way_table->copyMode ) + { + const char *tag_buf = pgsql_store_tags(tags,1); + char *node_buf = pgsql_store_nodes(nds); + int length = strlen(tag_buf) + strlen(node_buf) + 64; + buffer = (char *)alloca(length); + if( snprintf( buffer, length, "%" PRIdOSMID "\t%s\t%s\n", + way_id, node_buf, tag_buf ) > (length-10) ) + { fprintf( stderr, "buffer overflow way id %" PRIdOSMID "\n", way_id); return 1; } + pgsql_CopyData(__FUNCTION__, way_table->sql_conn, buffer); + return 0; + } + buffer = (char *)alloca(64); + char *ptr = buffer; + paramValues[0] = ptr; + sprintf(ptr, "%" PRIdOSMID, way_id); + paramValues[1] = pgsql_store_nodes(nds); + paramValues[2] = pgsql_store_tags(tags,0); + pgsql_execPrepared(way_table->sql_conn, "insert_way", 3, (const char * const *)paramValues, PGRES_COMMAND_OK); + return 0; +} + +int middle_pgsql_t::ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const +{ + char const *paramValues[1]; + PGconn *sql_conn = way_table->sql_conn; + + // Make sure we're out of copy mode */ + pgsql_endCopy( way_table ); + + char tmp[16]; + snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); + paramValues[0] = tmp; + + PGresult *res = pgsql_execPrepared(sql_conn, "get_way", 1, paramValues, PGRES_TUPLES_OK); + + if (PQntuples(res) != 1) { + PQclear(res); + return 1; + } + + pgsql_parse_tags( PQgetvalue(res, 0, 1), tags ); + + size_t num_nodes = strtoul(PQgetvalue(res, 0, 2), NULL, 10); + idlist_t list; + pgsql_parse_nodes( PQgetvalue(res, 0, 0), list); + if (num_nodes != list.size()) { + fprintf(stderr, "parse_nodes problem for way %s: expected nodes %zu got %zu\n", + tmp, num_nodes, list.size()); + util::exit_nicely(); + } + PQclear(res); + + nodes_get_list(nodes, list); + return 0; +} + +int middle_pgsql_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids, + multitaglist_t &tags, multinodelist_t &nodes) const { + if (ids.empty()) + return 0; + + char tmp[16]; + char *tmp2; + char const *paramValues[1]; + + tmp2 = (char *)malloc(sizeof(char)*ids.size()*16); + if (tmp2 == NULL) return 0; //failed to allocate memory, return */ + + // create a list of ids in tmp2 to query the database */ + sprintf(tmp2, "{"); + for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) { + snprintf(tmp, sizeof(tmp), "%" PRIdOSMID ",", *it); + strncat(tmp2,tmp, sizeof(char)*(ids.size()*16 - 2)); + } + tmp2[strlen(tmp2) - 1] = '}'; // replace last , with } to complete list of ids*/ + + pgsql_endCopy(way_table); + + PGconn *sql_conn = way_table->sql_conn; + + paramValues[0] = tmp2; + PGresult *res = pgsql_execPrepared(sql_conn, "get_way_list", 1, paramValues, PGRES_TUPLES_OK); + int countPG = PQntuples(res); + + idlist_t wayidspg; + + for (int i = 0; i < countPG; i++) { + wayidspg.push_back(strtoosmid(PQgetvalue(res, i, 0), NULL, 10)); + } + + + // Match the list of ways coming from postgres in a different order + // back to the list of ways given by the caller */ + for(idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) { + for (int j = 0; j < countPG; j++) { + if (*it == wayidspg[j]) { + way_ids.push_back(*it); + tags.push_back(taglist_t()); + pgsql_parse_tags(PQgetvalue(res, j, 2), tags.back()); + + size_t num_nodes = strtoul(PQgetvalue(res, j, 3), NULL, 10); + idlist_t list; + pgsql_parse_nodes( PQgetvalue(res, j, 1), list); + if (num_nodes != list.size()) { + fprintf(stderr, "parse_nodes problem for way %s: expected nodes %zu got %zu\n", + tmp, num_nodes, list.size()); + util::exit_nicely(); + } + + nodes.push_back(nodelist_t()); + nodes_get_list(nodes.back(), list); + + break; + } + } + } + + assert(way_ids.size() <= ids.size()); + + PQclear(res); + free(tmp2); + + return way_ids.size(); +} + + +int middle_pgsql_t::ways_delete(osmid_t osm_id) +{ + char const *paramValues[1]; + char buffer[64]; + // Make sure we're out of copy mode */ + pgsql_endCopy( way_table ); + + sprintf( buffer, "%" PRIdOSMID, osm_id ); + paramValues[0] = buffer; + pgsql_execPrepared(way_table->sql_conn, "delete_way", 1, paramValues, PGRES_COMMAND_OK ); + return 0; +} + +void middle_pgsql_t::iterate_ways(middle_t::pending_processor& pf) +{ + + // Make sure we're out of copy mode */ + pgsql_endCopy( way_table ); + + // enqueue the jobs + osmid_t id; + while(id_tracker::is_valid(id = ways_pending_tracker->pop_mark())) + { + pf.enqueue_ways(id); + } + // in case we had higher ones than the middle + pf.enqueue_ways(id_tracker::max()); + + //let the threads work on them + pf.process_ways(); +} + +int middle_pgsql_t::way_changed(osmid_t osm_id) +{ + char const *paramValues[1]; + char buffer[64]; + // Make sure we're out of copy mode */ + pgsql_endCopy( rel_table ); + + sprintf( buffer, "%" PRIdOSMID, osm_id ); + paramValues[0] = buffer; + + //keep track of whatever rels this way intersects + //TODO: dont need to stop the copy above since we are only reading? + PGresult* res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels_by_way", 1, paramValues, PGRES_TUPLES_OK ); + for(int i = 0; i < PQntuples(res); ++i) + { + char *end; + osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10); + rels_pending_tracker->mark(marked); + } + PQclear(res); + + return 0; +} + +int middle_pgsql_t::relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags) +{ + // Params: id, way_off, rel_off, parts, members, tags */ + const char *paramValues[6]; + char *buffer; + taglist_t member_list; + char buf[64]; + + idlist_t all_parts, node_parts, way_parts, rel_parts; + all_parts.reserve(members.size()); + node_parts.reserve(members.size()); + way_parts.reserve(members.size()); + rel_parts.reserve(members.size()); + + for (memberlist_t::const_iterator it = members.begin(); it != members.end(); ++it) { + char type = 0; + switch (it->type) + { + case OSMTYPE_NODE: node_parts.push_back(it->id); type = 'n'; break; + case OSMTYPE_WAY: way_parts.push_back(it->id); type = 'w'; break; + case OSMTYPE_RELATION: rel_parts.push_back(it->id); type = 'r'; break; + default: + fprintf(stderr, "Internal error: Unknown member type %d\n", it->type); + util::exit_nicely(); + } + sprintf( buf, "%c%" PRIdOSMID, type, it->id ); + member_list.push_back(tag(buf, it->role)); + } + + all_parts.insert(all_parts.end(), node_parts.begin(), node_parts.end()); + all_parts.insert(all_parts.end(), way_parts.begin(), way_parts.end()); + all_parts.insert(all_parts.end(), rel_parts.begin(), rel_parts.end()); + + if( rel_table->copyMode ) + { + char *tag_buf = strdup(pgsql_store_tags(tags,1)); + const char *member_buf = pgsql_store_tags(member_list,1); + char *parts_buf = pgsql_store_nodes(all_parts); + int length = strlen(member_buf) + strlen(tag_buf) + strlen(parts_buf) + 64; + buffer = (char *)alloca(length); + if( snprintf( buffer, length, "%" PRIdOSMID "\t%zu\t%zu\t%s\t%s\t%s\n", + id, node_parts.size(), node_parts.size() + way_parts.size(), + parts_buf, member_buf, tag_buf ) > (length-10) ) + { fprintf( stderr, "buffer overflow relation id %" PRIdOSMID "\n", id); return 1; } + free(tag_buf); + pgsql_CopyData(__FUNCTION__, rel_table->sql_conn, buffer); + return 0; + } + buffer = (char *)alloca(64); + char *ptr = buffer; + paramValues[0] = ptr; + ptr += sprintf(ptr, "%" PRIdOSMID, id ) + 1; + paramValues[1] = ptr; + ptr += sprintf(ptr, "%zu", node_parts.size() ) + 1; + paramValues[2] = ptr; + sprintf( ptr, "%zu", node_parts.size() + way_parts.size() ); + paramValues[3] = pgsql_store_nodes(all_parts); + paramValues[4] = pgsql_store_tags(member_list,0); + if( paramValues[4] ) + paramValues[4] = strdup(paramValues[4]); + paramValues[5] = pgsql_store_tags(tags,0); + pgsql_execPrepared(rel_table->sql_conn, "insert_rel", 6, (const char * const *)paramValues, PGRES_COMMAND_OK); + if( paramValues[4] ) + free((void *)paramValues[4]); + return 0; +} + +int middle_pgsql_t::relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const +{ + char tmp[16]; + char const *paramValues[1]; + PGconn *sql_conn = rel_table->sql_conn; + taglist_t member_temp; + + // Make sure we're out of copy mode */ + pgsql_endCopy( rel_table ); + + snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); + paramValues[0] = tmp; + + PGresult *res = pgsql_execPrepared(sql_conn, "get_rel", 1, paramValues, PGRES_TUPLES_OK); + // Fields are: members, tags, member_count */ + + if (PQntuples(res) != 1) { + PQclear(res); + return 1; + } + + pgsql_parse_tags(PQgetvalue(res, 0, 1), tags); + pgsql_parse_tags(PQgetvalue(res, 0, 0), member_temp); + + if (member_temp.size() != strtoul(PQgetvalue(res, 0, 2), NULL, 10)) { + fprintf(stderr, "Unexpected member_count reading relation %" PRIdOSMID "\n", id); + util::exit_nicely(); + } + + PQclear(res); + + for (taglist_t::const_iterator it = member_temp.begin(); it != member_temp.end(); ++it) { + char tag = it->key[0]; + OsmType type = (tag == 'n')?OSMTYPE_NODE:(tag == 'w')?OSMTYPE_WAY:(tag == 'r')?OSMTYPE_RELATION:((OsmType)-1); + members.push_back(member(type, + strtoosmid(it->key.c_str()+1, NULL, 10 ), + it->value)); + } + return 0; +} + +int middle_pgsql_t::relations_delete(osmid_t osm_id) +{ + char const *paramValues[1]; + char buffer[64]; + // Make sure we're out of copy mode */ + pgsql_endCopy( way_table ); + pgsql_endCopy( rel_table ); + + sprintf( buffer, "%" PRIdOSMID, osm_id ); + paramValues[0] = buffer; + pgsql_execPrepared(rel_table->sql_conn, "delete_rel", 1, paramValues, PGRES_COMMAND_OK ); + + //keep track of whatever ways this relation interesects + //TODO: dont need to stop the copy above since we are only reading? + PGresult* res = pgsql_execPrepared(way_table->sql_conn, "mark_ways_by_rel", 1, paramValues, PGRES_TUPLES_OK ); + for(int i = 0; i < PQntuples(res); ++i) + { + char *end; + osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10); + ways_pending_tracker->mark(marked); + } + PQclear(res); + return 0; +} + +void middle_pgsql_t::iterate_relations(pending_processor& pf) +{ + // Make sure we're out of copy mode */ + pgsql_endCopy( rel_table ); + + // enqueue the jobs + osmid_t id; + while(id_tracker::is_valid(id = rels_pending_tracker->pop_mark())) + { + pf.enqueue_relations(id); + } + // in case we had higher ones than the middle + pf.enqueue_relations(id_tracker::max()); + + //let the threads work on them + pf.process_relations(); +} + +int middle_pgsql_t::relation_changed(osmid_t osm_id) +{ + char const *paramValues[1]; + char buffer[64]; + // Make sure we're out of copy mode */ + pgsql_endCopy( rel_table ); + + sprintf( buffer, "%" PRIdOSMID, osm_id ); + paramValues[0] = buffer; + + //keep track of whatever ways and rels these nodes intersect + //TODO: dont need to stop the copy above since we are only reading? + //TODO: can we just mark the id without querying? the where clause seems intersect reltable.parts with the id + PGresult* res = pgsql_execPrepared(rel_table->sql_conn, "mark_rels", 1, paramValues, PGRES_TUPLES_OK ); + for(int i = 0; i < PQntuples(res); ++i) + { + char *end; + osmid_t marked = strtoosmid(PQgetvalue(res, i, 0), &end, 10); + rels_pending_tracker->mark(marked); + } + PQclear(res); + return 0; +} + +idlist_t middle_pgsql_t::relations_using_way(osmid_t way_id) const +{ + char const *paramValues[1]; + char buffer[64]; + // Make sure we're out of copy mode */ + pgsql_endCopy( rel_table ); + + sprintf(buffer, "%" PRIdOSMID, way_id); + paramValues[0] = buffer; + + PGresult *result = pgsql_execPrepared(rel_table->sql_conn, "rels_using_way", + 1, paramValues, PGRES_TUPLES_OK ); + const int ntuples = PQntuples(result); + idlist_t rel_ids(ntuples); + for (int i = 0; i < ntuples; ++i) { + rel_ids[i] = strtoosmid(PQgetvalue(result, i, 0), NULL, 10); + } + PQclear(result); + + return rel_ids; +} + +void middle_pgsql_t::analyze(void) +{ + for (int i=0; iprefix.empty()) { + strcpy(dest, options->prefix.c_str()); + dest += strlen(options->prefix.c_str()); + copied = 1; + } + source+=2; + continue; + } else if (*(source+1) == 't') { + if (options->tblsslim_data) { + strcpy(dest, options->tblsslim_data->c_str()); + dest += strlen(options->tblsslim_data->c_str()); + copied = 1; + } + source+=2; + continue; + } else if (*(source+1) == 'i') { + if (options->tblsslim_index) { + strcpy(dest, options->tblsslim_index->c_str()); + dest += strlen(options->tblsslim_index->c_str()); + copied = 1; + } + source+=2; + continue; + } else if (*(source+1) == 'm') { + if (options->unlogged) { + strcpy(dest, "UNLOGGED"); + dest += 8; + copied = 1; + } + source+=2; + continue; + } + } + *(dest++) = *(source++); + } + *dest = 0; + *string = strdup(buffer); +} + +int middle_pgsql_t::connect(table_desc& table) { + PGconn *sql_conn; + + set_prefix_and_tbls(out_options, &(table.name)); + set_prefix_and_tbls(out_options, &(table.start)); + set_prefix_and_tbls(out_options, &(table.create)); + set_prefix_and_tbls(out_options, &(table.create_index)); + set_prefix_and_tbls(out_options, &(table.prepare)); + set_prefix_and_tbls(out_options, &(table.prepare_intarray)); + set_prefix_and_tbls(out_options, &(table.copy)); + set_prefix_and_tbls(out_options, &(table.analyze)); + set_prefix_and_tbls(out_options, &(table.stop)); + set_prefix_and_tbls(out_options, &(table.array_indexes)); + + fprintf(stderr, "Setting up table: %s\n", table.name); + sql_conn = PQconnectdb(out_options->conninfo.c_str()); + + // Check to see that the backend connection was successfully made */ + if (PQstatus(sql_conn) != CONNECTION_OK) { + fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn)); + return 1; + } + table.sql_conn = sql_conn; + return 0; +} + +int middle_pgsql_t::start(const options_t *out_options_) +{ + out_options = out_options_; + PGresult *res; + int i; + int dropcreate = !out_options->append; + char * sql; + + ways_pending_tracker.reset(new id_tracker()); + rels_pending_tracker.reset(new id_tracker()); + + // Gazetter doesn't use mark-pending processing and consequently + // needs no way-node index. + // TODO Currently, set here to keep the impact on the code small. + // We actually should have the output plugins report their needs + // and pass that via the constructor to middle_t, so that middle_t + // itself doesn't need to know about details of the output. + if (out_options->output_backend == "gazetteer") { + way_table->array_indexes = NULL; + mark_pending = false; + } + + append = out_options->append; + // reset this on every start to avoid options from last run + // staying set for the second. + build_indexes = 0; + + cache.reset(new node_ram_cache( out_options->alloc_chunkwise | ALLOC_LOSSY, out_options->cache, out_options->scale)); + if (out_options->flat_node_cache_enabled) persistent_cache.reset(new node_persistent_cache(out_options, out_options->append, false, cache)); + + fprintf(stderr, "Mid: pgsql, scale=%d cache=%d\n", out_options->scale, out_options->cache); + + // We use a connection per table to enable the use of COPY */ + for (i=0; iappend) + { + sql = (char *)malloc (2048); + snprintf(sql, 2047, "SELECT id FROM %s LIMIT 1", tables[t_node].name); + res = PQexec(sql_conn, sql ); + free(sql); + sql = NULL; + if(PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1) + { + int size = PQfsize(res, 0); + if (size != sizeof(osmid_t)) + { + fprintf(stderr, + "\n" + "The target database has been created with %dbit ID fields,\n" + "but this version of osm2pgsql has been compiled to use %ldbit IDs.\n" + "You cannot append data to this database with this program.\n" + "Either re-create the database or use a matching osm2pgsql.\n\n", + size * 8, sizeof(osmid_t) * 8); + util::exit_nicely(); + } + } + PQclear(res); + } + + if(!out_options->append) + build_indexes = 1; + } + + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET client_min_messages = WARNING"); + if (dropcreate) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s", tables[i].name); + } + + if (tables[i].start) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].start); + tables[i].transactionMode = 1; + } + + if (dropcreate && tables[i].create) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create); + if (tables[i].create_index) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].create_index); + } + } + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "RESET client_min_messages"); + + if (tables[i].prepare) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare); + } + + if (append && tables[i].prepare_intarray) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray); + } + + if (tables[i].copy) { + pgsql_exec(sql_conn, PGRES_COPY_IN, "%s", tables[i].copy); + tables[i].copyMode = 1; + } + } + + return 0; +} + +void middle_pgsql_t::commit(void) { + int i; + for (i=0; iflat_node_cache_enabled) persistent_cache.reset(); +} + +void *middle_pgsql_t::pgsql_stop_one(void *arg) +{ + time_t start, end; + + struct table_desc *table = (struct table_desc *)arg; + PGconn *sql_conn = table->sql_conn; + + fprintf(stderr, "Stopping table: %s\n", table->name); + pgsql_endCopy(table); + time(&start); + if (out_options->droptemp) + { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE %s", table->name); + } + else if (build_indexes && table->array_indexes) + { + fprintf(stderr, "Building index on table: %s\n", table->name); + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", table->array_indexes); + } + + PQfinish(sql_conn); + table->sql_conn = NULL; + time(&end); + fprintf(stderr, "Stopped table: %s in %is\n", table->name, (int)(end - start)); + return NULL; +} + +namespace { +/* Using pthreads requires us to shoe-horn everything into various void* + * pointers. Improvement for the future: just use boost::thread. */ +struct pthread_thunk { + middle_pgsql_t *obj; + void *ptr; +}; + +extern "C" void *pthread_middle_pgsql_stop_one(void *arg) { + pthread_thunk *thunk = static_cast(arg); + return thunk->obj->pgsql_stop_one(thunk->ptr); +} +} // anonymous namespace + +void middle_pgsql_t::stop(void) +{ + int i; +#ifdef HAVE_PTHREAD + pthread_t threads[num_tables]; +#endif + + cache.reset(); + if (out_options->flat_node_cache_enabled) persistent_cache.reset(); + +#ifdef HAVE_PTHREAD + pthread_thunk thunks[num_tables]; + for (i=0; i middle_pgsql_t::get_instance() const { + middle_pgsql_t* mid = new middle_pgsql_t(); + mid->out_options = out_options; + mid->append = out_options->append; + mid->mark_pending = mark_pending; + + //NOTE: this is thread safe for use in pending async processing only because + //during that process they are only read from + mid->cache = cache; + // The persistent cache on the other hand is not thread-safe for reading, + // so we create one per instance. + if (out_options->flat_node_cache_enabled) + mid->persistent_cache.reset(new node_persistent_cache(out_options, 1, true, cache)); + + // We use a connection per table to enable the use of COPY */ + for(int i=0; iconnect(mid->tables[i])) + util::exit_nicely(); + PGconn* sql_conn = mid->tables[i].sql_conn; + + if (tables[i].prepare) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare); + } + + if (append && tables[i].prepare_intarray) { + pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", tables[i].prepare_intarray); + } + } + + return boost::shared_ptr(mid); +} + +size_t middle_pgsql_t::pending_count() const { + return ways_pending_tracker->size() + rels_pending_tracker->size(); +} diff -Nru osm2pgsql-0.82.0/middle-pgsql.h osm2pgsql-0.88.1/middle-pgsql.h --- osm2pgsql-0.82.0/middle-pgsql.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/middle-pgsql.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -/* Implements the mid-layer processing for osm2pgsql - * using several PostgreSQL tables - * - * This layer stores data read in from the planet.osm file - * and is then read by the backend processing code to - * emit the final geometry-enabled output formats -*/ - -#ifndef MIDDLE_PGSQL_H -#define MIDDLE_PGSQL_H - -extern struct middle_t mid_pgsql; - -#endif diff -Nru osm2pgsql-0.82.0/middle-pgsql.hpp osm2pgsql-0.88.1/middle-pgsql.hpp --- osm2pgsql-0.82.0/middle-pgsql.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/middle-pgsql.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,108 @@ +/* Implements the mid-layer processing for osm2pgsql + * using several PostgreSQL tables + * + * This layer stores data read in from the planet.osm file + * and is then read by the backend processing code to + * emit the final geometry-enabled output formats +*/ + +#ifndef MIDDLE_PGSQL_H +#define MIDDLE_PGSQL_H + +#include "middle.hpp" +#include "node-ram-cache.hpp" +#include "node-persistent-cache.hpp" +#include "id-tracker.hpp" +#include +#include +#include + +struct middle_pgsql_t : public slim_middle_t { + middle_pgsql_t(); + virtual ~middle_pgsql_t(); + + int start(const options_t *out_options_); + void stop(void); + void analyze(void); + void end(void); + void commit(void); + + int nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags); + int nodes_get_list(nodelist_t &out, const idlist_t nds) const; + int nodes_delete(osmid_t id); + int node_changed(osmid_t id); + + int ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags); + int ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const; + int ways_get_list(const idlist_t &ids, idlist_t &way_ids, + multitaglist_t &tags, multinodelist_t &nodes) const; + + int ways_delete(osmid_t id); + int way_changed(osmid_t id); + + int relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const; + int relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags); + int relations_delete(osmid_t id); + int relation_changed(osmid_t id); + + void iterate_ways(middle_t::pending_processor& pf); + void iterate_relations(pending_processor& pf); + + size_t pending_count() const; + + std::vector relations_using_way(osmid_t way_id) const; + + void *pgsql_stop_one(void *arg); + + struct table_desc { + table_desc(const char *name_ = NULL, + const char *start_ = NULL, + const char *create_ = NULL, + const char *create_index_ = NULL, + const char *prepare_ = NULL, + const char *prepare_intarray_ = NULL, + const char *copy_ = NULL, + const char *analyze_ = NULL, + const char *stop_ = NULL, + const char *array_indexes_ = NULL); + + const char *name; + const char *start; + const char *create; + const char *create_index; + const char *prepare; + const char *prepare_intarray; + const char *copy; + const char *analyze; + const char *stop; + const char *array_indexes; + + int copyMode; /* True if we are in copy mode */ + int transactionMode; /* True if we are in an extended transaction */ + struct pg_conn *sql_conn; + }; + + virtual boost::shared_ptr get_instance() const; +private: + + int connect(table_desc& table); + int local_nodes_set(const osmid_t& id, const double& lat, const double& lon, const taglist_t &tags); + int local_nodes_get_list(nodelist_t &out, const idlist_t nds) const; + int local_nodes_delete(osmid_t osm_id); + + std::vector tables; + int num_tables; + table_desc *node_table, *way_table, *rel_table; + + int append; + bool mark_pending; + + boost::shared_ptr cache; + boost::shared_ptr persistent_cache; + + boost::shared_ptr ways_pending_tracker, rels_pending_tracker; + + int build_indexes; +}; + +#endif diff -Nru osm2pgsql-0.82.0/middle-ram.c osm2pgsql-0.88.1/middle-ram.c --- osm2pgsql-0.82.0/middle-ram.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/middle-ram.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,388 +0,0 @@ -/* Implements the mid-layer processing for osm2pgsql - * using several arrays in RAM. This is fastest if you - * have sufficient RAM+Swap. - * - * This layer stores data read in from the planet.osm file - * and is then read by the backend processing code to - * emit the final geometry-enabled output formats -*/ - -#include -#include -#include -#include -#include -#include - -#include "osmtypes.h" -#include "middle.h" -#include "middle-ram.h" -#include "node-ram-cache.h" - -#include "output-pgsql.h" - -/* Store +-20,000km Mercator co-ordinates as fixed point 32bit number with maximum precision */ -/* Scale is chosen such that 40,000 * SCALE < 2^32 */ -#define FIXED_POINT - -struct ramWay { - struct keyval *tags; - osmid_t *ndids; - int pending; -}; - -struct ramRel { - struct keyval *tags; - struct member *members; - int member_count; -}; - -/* Object storage now uses 2 levels of storage arrays. - * - * - Low level storage of 2^16 (~65k) objects in an indexed array - * These are allocated dynamically when we need to first store data with - * an ID in this block - * - * - Fixed array of 2^(32 - 16) = 65k pointers to the dynamically allocated arrays. - * - * This allows memory usage to be efficient and scale dynamically without needing to - * hard code maximum IDs. We now support an ID range of -2^31 to +2^31. - * The negative IDs often occur in non-uploaded JOSM data or other data import scripts. - * - */ - -#define BLOCK_SHIFT 10 -#define PER_BLOCK (1 << BLOCK_SHIFT) -#define NUM_BLOCKS (1 << (32 - BLOCK_SHIFT)) - -static struct ramWay *ways[NUM_BLOCKS]; -static struct ramRel *rels[NUM_BLOCKS]; - -static int node_blocks; -static int way_blocks; - -static int way_out_count; -static int rel_out_count; - -static osmid_t id2block(osmid_t id) -{ - /* + NUM_BLOCKS/2 allows for negative IDs */ - return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2; -} - -static osmid_t id2offset(osmid_t id) -{ - return id & (PER_BLOCK-1); -} - -static int block2id(int block, int offset) -{ - return ((block - NUM_BLOCKS/2) << BLOCK_SHIFT) + offset; -} - -#define UNUSED __attribute__ ((unused)) - -static int ram_ways_set(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags, int pending) -{ - int block = id2block(id); - int offset = id2offset(id); - struct keyval *p; - - if (!ways[block]) { - ways[block] = calloc(PER_BLOCK, sizeof(struct ramWay)); - if (!ways[block]) { - fprintf(stderr, "Error allocating ways\n"); - exit_nicely(); - } - way_blocks++; - } - - if (ways[block][offset].ndids) { - free(ways[block][offset].ndids); - ways[block][offset].ndids = NULL; - } - - /* Copy into length prefixed array */ - ways[block][offset].ndids = malloc( (nd_count+1)*sizeof(osmid_t) ); - memcpy( ways[block][offset].ndids+1, nds, nd_count*sizeof(osmid_t) ); - ways[block][offset].ndids[0] = nd_count; - ways[block][offset].pending = pending; - - if (!ways[block][offset].tags) { - p = malloc(sizeof(struct keyval)); - if (p) { - initList(p); - ways[block][offset].tags = p; - } else { - fprintf(stderr, "%s malloc failed\n", __FUNCTION__); - exit_nicely(); - } - } else - resetList(ways[block][offset].tags); - - cloneList(ways[block][offset].tags, tags); - - return 0; -} - -static int ram_relations_set(osmid_t id, struct member *members, int member_count, struct keyval *tags) -{ - struct keyval *p; - struct member *ptr; - int block = id2block(id); - int offset = id2offset(id); - if (!rels[block]) { - rels[block] = calloc(PER_BLOCK, sizeof(struct ramRel)); - if (!rels[block]) { - fprintf(stderr, "Error allocating rels\n"); - exit_nicely(); - } - } - - if (!rels[block][offset].tags) { - p = malloc(sizeof(struct keyval)); - if (p) { - initList(p); - rels[block][offset].tags = p; - } else { - fprintf(stderr, "%s malloc failed\n", __FUNCTION__); - exit_nicely(); - } - } else - resetList(rels[block][offset].tags); - - cloneList(rels[block][offset].tags, tags); - - if (!rels[block][offset].members) - free( rels[block][offset].members ); - - ptr = malloc(sizeof(struct member) * member_count); - if (ptr) { - memcpy( ptr, members, sizeof(struct member) * member_count ); - rels[block][offset].member_count = member_count; - rels[block][offset].members = ptr; - } else { - fprintf(stderr, "%s malloc failed\n", __FUNCTION__); - exit_nicely(); - } - - return 0; -} - -static int ram_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, int nd_count) -{ - int i, count; - - count = 0; - for( i=0; i=0; block--) { - if (!rels[block]) - continue; - - for (offset=0; offset < PER_BLOCK; offset++) { - if (rels[block][offset].members) { - osmid_t id = block2id(block, offset); - rel_out_count++; - if (rel_out_count % 10 == 0) - fprintf(stderr, "\rWriting relation (%u)", rel_out_count); - - callback(id, rels[block][offset].members, rels[block][offset].member_count, rels[block][offset].tags, 0); - } - free(rels[block][offset].members); - rels[block][offset].members = NULL; - resetList(rels[block][offset].tags); - free(rels[block][offset].tags); - rels[block][offset].tags=NULL; - } - free(rels[block]); - rels[block] = NULL; - } - - fprintf(stderr, "\rWriting relation (%u)\n", rel_out_count); -} - -static void ram_iterate_ways(int (*callback)(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists)) -{ - int block, offset, ndCount = 0; - struct osmNode *nodes; - - fprintf(stderr, "\n"); - for(block=NUM_BLOCKS-1; block>=0; block--) { - if (!ways[block]) - continue; - - for (offset=0; offset < PER_BLOCK; offset++) { - if (ways[block][offset].ndids) { - way_out_count++; - if (way_out_count % 1000 == 0) - fprintf(stderr, "\rWriting way (%uk)", way_out_count/1000); - - if (ways[block][offset].pending) { - /* First element contains number of nodes */ - nodes = malloc( sizeof(struct osmNode) * ways[block][offset].ndids[0]); - ndCount = ram_nodes_get_list(nodes, ways[block][offset].ndids+1, ways[block][offset].ndids[0]); - - if (nodes) { - osmid_t id = block2id(block, offset); - callback(id, ways[block][offset].tags, nodes, ndCount, 0); - free(nodes); - } - - ways[block][offset].pending = 0; - } - - if (ways[block][offset].tags) { - resetList(ways[block][offset].tags); - free(ways[block][offset].tags); - ways[block][offset].tags = NULL; - } - if (ways[block][offset].ndids) { - free(ways[block][offset].ndids); - ways[block][offset].ndids = NULL; - } - } - } - } - fprintf(stderr, "\rWriting way (%uk)\n", way_out_count/1000); -} - -/* Caller must free nodes_ptr and resetList(tags_ptr) */ -static int ram_ways_get(osmid_t id, struct keyval *tags_ptr, struct osmNode **nodes_ptr, int *count_ptr) -{ - int block = id2block(id), offset = id2offset(id), ndCount = 0; - struct osmNode *nodes; - - if (!ways[block]) - return 1; - - if (ways[block][offset].ndids) { - /* First element contains number of nodes */ - nodes = malloc( sizeof(struct osmNode) * ways[block][offset].ndids[0]); - ndCount = ram_nodes_get_list(nodes, ways[block][offset].ndids+1, ways[block][offset].ndids[0]); - - if (ndCount) { - cloneList( tags_ptr, ways[block][offset].tags ); - *nodes_ptr = nodes; - *count_ptr = ndCount; - return 0; - } - free(nodes); - } - return 1; -} - -static int ram_ways_get_list(osmid_t *ids, int way_count, osmid_t **way_ids, struct keyval *tag_ptr, struct osmNode **node_ptr, int *count_ptr) { - int count = 0; - int i; - *way_ids = malloc( sizeof(osmid_t) * (way_count + 1)); - initList(&(tag_ptr[count])); - for (i = 0; i < way_count; i++) { - - if (ram_ways_get(ids[i], &(tag_ptr[count]), &(node_ptr[count]), &(count_ptr[count])) == 0) { - (*way_ids)[count] = ids[i]; - count++; - initList(&(tag_ptr[count])); - } - } - return count; -} - -/* Marks the way so that iterate ways skips it */ -static int ram_ways_done(osmid_t id) -{ - int block = id2block(id), offset = id2offset(id); - - if (!ways[block]) - return 1; - - ways[block][offset].pending = 0; - return 0; -} - -static void ram_analyze(void) -{ - /* No need */ -} - -static void ram_end(void) -{ - /* No need */ -} - -static int ram_start(const struct output_options *options) -{ - /* latlong has a range of +-180, mercator +-20000 - The fixed poing scaling needs adjusting accordingly to - be stored accurately in an int */ - scale = options->scale; - - init_node_ram_cache( options->alloc_chunkwise, options->cache, scale); - - fprintf( stderr, "Mid: Ram, scale=%d\n", scale ); - - return 0; -} - -static void ram_stop(void) -{ - int i, j; - free_node_ram_cache(); - - for (i=0; i +#include +#include +#include + +#include "middle-ram.hpp" +#include "node-ram-cache.hpp" +#include "options.hpp" +#include "id-tracker.hpp" + +/* Object storage now uses 2 levels of storage arrays. + * + * - Low level storage of 2^16 (~65k) objects in an indexed array + * These are allocated dynamically when we need to first store data with + * an ID in this block + * + * - Fixed array of 2^(32 - 16) = 65k pointers to the dynamically allocated arrays. + * + * This allows memory usage to be efficient and scale dynamically without needing to + * hard code maximum IDs. We now support an ID range of -2^31 to +2^31. + * The negative IDs often occur in non-uploaded JOSM data or other data import scripts. + * + */ + +#define BLOCK_SHIFT 10 +#define PER_BLOCK (1 << BLOCK_SHIFT) +#define NUM_BLOCKS (1 << (32 - BLOCK_SHIFT)) + +static osmid_t id2block(osmid_t id) +{ + /* + NUM_BLOCKS/2 allows for negative IDs */ + return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2; +} + +static osmid_t id2offset(osmid_t id) +{ + return id & (PER_BLOCK-1); +} + +int middle_ram_t::nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags) { + return cache->set(id, lat, lon, tags); +} + +int middle_ram_t::ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags) +{ + int block = id2block(id); + int offset = id2offset(id); + + if (ways[block].empty()) { + ways[block].assign(PER_BLOCK, ramWay()); + } + + ways[block][offset].ndids = nds; + ways[block][offset].tags = tags; + + return 0; +} + +int middle_ram_t::relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags) +{ + int block = id2block(id); + int offset = id2offset(id); + if (rels[block].empty()) { + rels[block].assign(PER_BLOCK, ramRel()); + } + + rels[block][offset].tags = tags; + rels[block][offset].members = members; + + return 0; +} + +int middle_ram_t::nodes_get_list(nodelist_t &out, const idlist_t nds) const +{ + for (idlist_t::const_iterator it = nds.begin(); it != nds.end(); ++it) { + osmNode n; + if (!cache->get(&n, *it)) + out.push_back(n); + } + + return out.size(); +} + +void middle_ram_t::iterate_relations(pending_processor& pf) +{ + //TODO: just dont do anything + + //let the outputs enqueue everything they have the non slim middle + //has nothing of its own to enqueue as it doesnt have pending anything + pf.enqueue_relations(id_tracker::max()); + + //let the threads process the relations + pf.process_relations(); +} + +size_t middle_ram_t::pending_count() const +{ + return 0; +} + +void middle_ram_t::iterate_ways(middle_t::pending_processor& pf) +{ + //let the outputs enqueue everything they have the non slim middle + //has nothing of its own to enqueue as it doesnt have pending anything + pf.enqueue_ways(id_tracker::max()); + + //let the threads process the ways + pf.process_ways(); +} + +void middle_ram_t::release_relations() +{ + rels.clear(); +} + +void middle_ram_t::release_ways() +{ + ways.clear(); +} + +int middle_ram_t::ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const +{ + if (simulate_ways_deleted) + return 1; + + int block = id2block(id), offset = id2offset(id); + + if (ways[block].empty() || ways[block][offset].ndids.empty()) + return 1; + + tags = ways[block][offset].tags; + + nodes_get_list(nodes, ways[block][offset].ndids); + + return 0; +} + +int middle_ram_t::ways_get_list(const idlist_t &ids, idlist_t &way_ids, + multitaglist_t &tags, multinodelist_t &nodes) const +{ + if (ids.empty()) + return 0; + + assert(way_ids.empty()); + tags.assign(ids.size(), taglist_t()); + nodes.assign(ids.size(), nodelist_t()); + + size_t count = 0; + for (idlist_t::const_iterator it = ids.begin(); it != ids.end(); ++it) { + if (ways_get(*it, tags[count], nodes[count]) == 0) { + way_ids.push_back(*it); + count++; + } else { + tags[count].clear(); + nodes[count].clear(); + } + } + + if (count < ids.size()) { + tags.resize(count); + nodes.resize(count); + } + + return count; +} + +int middle_ram_t::relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const +{ + int block = id2block(id), offset = id2offset(id); + + if (rels[block].empty() || rels[block][offset].members.empty()) + return 1; + + members = rels[block][offset].members; + tags = rels[block][offset].tags; + + return 0; +} + +void middle_ram_t::analyze(void) +{ + /* No need */ +} + +void middle_ram_t::end(void) +{ + /* No need */ +} + +int middle_ram_t::start(const options_t *out_options_) +{ + out_options = out_options_; + /* latlong has a range of +-180, mercator +-20000 + The fixed poing scaling needs adjusting accordingly to + be stored accurately in an int */ + cache.reset(new node_ram_cache(out_options->alloc_chunkwise, out_options->cache, out_options->scale)); + + fprintf( stderr, "Mid: Ram, scale=%d\n", out_options->scale ); + + return 0; +} + +void middle_ram_t::stop(void) +{ + cache.reset(NULL); + + release_ways(); + release_relations(); +} + +void middle_ram_t::commit(void) { +} + +middle_ram_t::middle_ram_t(): + ways(), rels(), cache(), + simulate_ways_deleted(false) +{ + ways.resize(NUM_BLOCKS); memset(&ways[0], 0, NUM_BLOCKS * sizeof ways[0]); + rels.resize(NUM_BLOCKS); memset(&rels[0], 0, NUM_BLOCKS * sizeof rels[0]); +} + +middle_ram_t::~middle_ram_t() { + //instance.reset(); +} + +std::vector middle_ram_t::relations_using_way(osmid_t way_id) const +{ + // this function shouldn't be called - relations_using_way is only used in + // slim mode, and a middle_ram_t shouldn't be constructed if the slim mode + // option is set. + throw std::runtime_error("middle_ram_t::relations_using_way is unimlpemented, and " + "should not have been called. This is probably a bug, please " + "report it at https://github.com/openstreetmap/osm2pgsql/issues"); +} + +namespace { + +void no_delete(const middle_ram_t * middle) { + // boost::shared_ptr thinks we are going to delete + // the middle object, but we are not. Heh heh heh. + // So yeah, this is a hack... +} + +} + +boost::shared_ptr middle_ram_t::get_instance() const { + //shallow copy here because readonly access is thread safe + return boost::shared_ptr(this, no_delete); +} diff -Nru osm2pgsql-0.82.0/middle-ram.h osm2pgsql-0.88.1/middle-ram.h --- osm2pgsql-0.82.0/middle-ram.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/middle-ram.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -/* Implements the mid-layer processing for osm2pgsql - * using several PostgreSQL tables - * - * This layer stores data read in from the planet.osm file - * and is then read by the backend processing code to - * emit the final geometry-enabled output formats -*/ - -#ifndef MIDDLE_RAM_H -#define MIDDLE_RAM_H - -extern struct middle_t mid_ram; - -#endif diff -Nru osm2pgsql-0.82.0/middle-ram.hpp osm2pgsql-0.88.1/middle-ram.hpp --- osm2pgsql-0.82.0/middle-ram.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/middle-ram.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,86 @@ +/* Implements the mid-layer processing for osm2pgsql + * using data structures in RAM. + * + * This layer stores data read in from the planet.osm file + * and is then read by the backend processing code to + * emit the final geometry-enabled output formats +*/ + +#ifndef MIDDLE_RAM_H +#define MIDDLE_RAM_H + +#include + +#include "middle.hpp" +#include + +struct node_ram_cache; +struct options_t; + +struct middle_ram_t : public middle_t { + middle_ram_t(); + virtual ~middle_ram_t(); + + int start(const options_t *out_options_); + void stop(void); + void analyze(void); + void end(void); + void commit(void); + + int nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags); + int nodes_get_list(nodelist_t &out, const idlist_t nds) const; + int nodes_delete(osmid_t id); + int node_changed(osmid_t id); + + int ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags); + int ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const; + int ways_get_list(const idlist_t &ids, idlist_t &way_ids, + multitaglist_t &tags, multinodelist_t &nodes) const; + + int ways_delete(osmid_t id); + int way_changed(osmid_t id); + + int relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const; + int relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags); + int relations_delete(osmid_t id); + int relation_changed(osmid_t id); + + std::vector relations_using_way(osmid_t way_id) const; + + void iterate_ways(middle_t::pending_processor& pf); + void iterate_relations(pending_processor& pf); + + size_t pending_count() const; + + virtual boost::shared_ptr get_instance() const; +private: + + void release_ways(); + void release_relations(); + + struct ramWay { + taglist_t tags; + idlist_t ndids; + }; + + struct ramRel { + taglist_t tags; + memberlist_t members; + }; + + std::vector > ways; + std::vector > rels; + + std::auto_ptr cache; + + /* the previous behaviour of iterate_ways was to delete all ways as they + * were being iterated. this doesn't work now that the output handles its + * own "done" status and output-specific "pending" status. however, the + * tests depend on the behaviour that ways will be unavailable once + * iterate_ways is complete, so this flag emulates that. */ + bool simulate_ways_deleted; +}; + +extern middle_ram_t mid_ram; + +#endif diff -Nru osm2pgsql-0.82.0/multi.lua osm2pgsql-0.88.1/multi.lua --- osm2pgsql-0.82.0/multi.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/multi.lua 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,271 @@ +-- This is an example Lua transform for a multi style +-- It is not intended for use directly with --tag-transform-script but +-- for use from multi.style.json +-- +-- See docs/lua.md and docs/multi.md + +-- These are copied from default.style +-- If new "tags" are being generated in the Lua code they should normally be +-- added here. This is why name_.* is dropped. In the raw OSM data +-- multi-lingual names are stored in name:*. + +delete_tags = {'name_.*', 'note', 'note:.*', 'source', 'source_ref', 'source:.*', + 'attribution', 'comment', 'fixme', 'created_by', 'odbl', + 'odbl:note', 'SK53_bulk:load', 'tiger:.*', 'NHD:.*', 'nhd:.*', + 'gnis:.*', 'geobase:.*', 'accuracy:meters', 'sub_sea:type', + 'waterway:type', 'KSJ2:.*', 'yh:.*', 'osak:.*', 'kms:.*', 'ngbe:.*', + 'naptan:.*', 'CLC:.*', '3dshapes:ggmodelk', 'AND_nosr_r', + 'import', 'it:fvg:.*'} + +-- In a real transform the Lua code might be split into multiple files with +-- common code included with "dofile" but a single file is easier for an example + +-- A function to determine if the tags make the object "interesting" to the +-- buildings table +function building_interesting (kv) + return kv["building"] and kv["building"] ~= "no" +end + +function building_transform (kv) + kv["name_en"] = name_lang(kv, "en") + kv["name_de"] = name_lang(kv, "de") + kv["name_fr"] = name_lang(kv, "fr") + return kv +end + +-- If we weren't generating multilingual names we could omit building_transform +function building_ways (kv, num_keys) + return generic_ways(building_interesting, kv, true, building_transform) +end + +function building_rels (kv, num_keys) + return generic_rels(building_interesting, kv) +end + +function building_rel_members (kv, keyvaluemembers, roles, membercount) + return generic_rel_members(building_interesting, kv, keyvaluemembers, roles, membercount, building_transform) +end + +-- A function to determine if the tags make the object "interesting" to the +-- bus stop table +function bus_interesting (kv) + return kv["highway"] == "bus_stop" +end + +function bus_transform (kv) + kv["shelter"] = yesno(kv["shelter"]) + kv["bench"] = yesno(kv["bench"]) + kv["wheelchair"] = yesno(kv["wheelchair"]) + kv["name_en"] = name_lang(kv, "en") + kv["name_de"] = name_lang(kv, "de") + kv["name_fr"] = name_lang(kv, "fr") + return kv +end +function bus_nodes (kv, num_keys) + return generic_nodes(bus_interesting, kv, bus_transform) +end + +-- lookup tables for highways. Using an enum would be better in some ways, but +-- would require creating the type before importing with osm2pgsql, which is +-- not well suited to an example. + +highway_lookup = {motorway = 0, + trunk = 1, + primary = 2, + secondary = 3, + tertiary = 4, + unclassified = 5, + residential = 5} + +link_lookup = {motorway_link = 0, + trunk_link = 1, + primary_link = 2, + secondary_link = 3, + tertiary_link = 4} + +function highway_interesting (kv) + -- The kv["highway"] check is not necessary but helps performance + return kv["highway"] and (highway_lookup[kv["highway"]] or link_lookup[kv["highway"]]) +end + +function highway_transform (kv) + -- Thanks to highway_interesting we know that kv["highway"] is in one of + -- highway_lookup or link_lookup + kv["road_class"] = highway_lookup[kv["highway"]] or link_lookup[kv["highway"]] + -- This is a lua way of doing an inline conditional + kv["road_type"] = highway_lookup[kv["highway"]] and "road" or "link" + kv["name_en"] = name_lang(kv, "en") + kv["name_de"] = name_lang(kv, "de") + kv["name_fr"] = name_lang(kv, "fr") + return kv +end + +function highway_ways (kv, num_keys) + return generic_ways(highway_interesting, kv, false, highway_transform) +end + +-- Some generic and utility helper functions + +-- This function normalizes a tag to true/false. It turns no or false into +-- false and anything else to true. The result can then be used with a +-- boolean column. +-- > = yesno(nil) +-- nil +-- > = yesno("no") +-- false +-- > = yesno("false") +-- false +-- > = yesno("yes") +-- true +-- > = yesno("foo") +-- true +-- +-- A typical usage would be on a tag like bridge, tunnel, or shelter, but not +-- a tag like oneway which could be yes, no, reverse, or unset +function yesno (v) + -- This is a way of doing an inline condition in Lua + return v ~= nil and ((v == "no" or v == "false") and "false" or "true") or nil +end + +-- Converts a name and name:lang tag into one combined name +-- By passing an optional name_tag parameter it can also work with other +-- multi-lingual tags +function name_lang(kv, lang, name_tag) + if kv then + -- Default to the name tag, which is what this will generally be used on. + name_tag = name_tag or "name" + -- Defaulting to en is a bit of complete Anglo-centrism + lang = lang or "en" + name = kv[name_tag] + name_trans = kv[name_tag .. ":" .. lang] + -- If we don't have a translated name, just use the name (which may be blank) + if not name_trans then return name end + -- If we do have a translated name and not a local language name, use the translated + if not name then return name_trans end + -- if they're the same, return one of them + if name == name_trans then return name end + -- This method presents some problems when multiple names get put in the + -- name tag. + return name_trans .. "(" .. name .. ")" + end +end + +-- This function gets rid of an object we don't care about +function drop_all (...) + return 1, {} +end + +-- This eliminates tags to be deleted +function preprocess_tags (kv) + tags = {} + for k, v in pairs (kv) do + match = false + for _, d in ipairs(delete_tags) do + match = match or string.find(k, d) + end + if not match then + tags[k] = v + end + end + return tags +end + +-- A generic way to process nodes, given a function which determines if tags are interesting +-- Takes an optional function to process tags +function generic_nodes (f, kv, t) + if f(kv) then + t = t or function (kv) return kv end + tags = t(kv) + return 0, tags + else + return 1, {} + end +end + + +-- A generic way to process ways, given a function which determines if tags are interesting +-- Takes an optional function to process tags. +function generic_ways (interesting, kv, area, transform) + if interesting(kv) then + t = transform or function (kv) return kv end + tags = t(preprocess_tags(kv)) + return 0, tags, area and 1 or 0, 0 + else + return 1, {}, 0, 0 + end +end + +-- A generic way to process relations, given a function which determines if +-- tags are interesting. The tag transformation work is done in +-- generic_rel_members so we don't need to pass in a transformation function. +function generic_rels (f, kv) + if kv["type"] == "multipolygon" and f(kv) then + tags = kv + return 0, tags + else + return 1, {} + end +end + +-- Basically taken from style.lua, with the potential for a transform added +function generic_rel_members (f, keyvals, keyvaluemembers, roles, membercount, transform) + filter = 0 + boundary = 0 + polygon = 0 + roads = 0 + t = transform or function (kv) return kv end + + --mark each way of the relation to tell the caller if its going + --to be used in the relation or by itself as its own standalone way + --we start by assuming each way will not be used as part of the relation + membersuperseeded = {} + for i = 1, membercount do + membersuperseeded[i] = 0 + end + + --remember the type on the relation and erase it from the tags + type = keyvals["type"] + keyvals["type"] = nil + + if (type == "multipolygon") and keyvals["boundary"] == nil then + --check if this relation has tags we care about + polygon = 1 + filter = f(keyvals) + + --if the relation didn't have the tags we need go grab the tags from + --any members that are marked as outers of the multipolygon + if (filter == 1) then + for i = 1,membercount do + if (roles[i] == "outer") then + for j,k in ipairs(tags) do + v = keyvaluemembers[i][k] + if v then + keyvals[k] = v + filter = 0 + end + end + end + end + end + if filter == 1 then + tags = t(keyvals) + return filter, tags, membersuperseeded, boundary, polygon, roads + end + + --for each tag of each member if the relation have the tag or has a non matching value for it + --then we say the member will not be used in the relation and is there for not superseeded + --ie it is kept as a standalone way + for i = 1,membercount do + superseeded = 1 + for k,v in pairs(keyvaluemembers[i]) do + if ((keyvals[k] == nil) or (keyvals[k] ~= v)) then + superseeded = 0; + break + end + end + membersuperseeded[i] = superseeded + end + end + tags = t(keyvals) + return filter, tags, membersuperseeded, boundary, polygon, roads +end diff -Nru osm2pgsql-0.82.0/multi.style.json osm2pgsql-0.88.1/multi.style.json --- osm2pgsql-0.82.0/multi.style.json 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/multi.style.json 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,67 @@ +/* This is an example multi-backend style file that comes with osm2pgsql. + * + * It contains comments so is invalid JSON but the parser doesn't complain. + * + * To use this file, pass it in as a style file with the --style switch, e.g. + * with osm2pgsql --output multi --style multi.style.json + */ +[ + { + "name": "stops", + "type": "point", + "tagtransform": "multi.lua", + "tagtransform-node-function": "bus_nodes", + "tagtransform-way-function": "drop_all", + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + /* We don't need a column for highway, since it's always highway=bus_stop */ + {"name": "name", "type": "text"}, + /* multi-lingual names can be generated at import time */ + {"name": "name_en", "type": "text"}, + {"name": "name_fr", "type": "text"}, + {"name": "name_de", "type": "text"}, + {"name": "ref", "type": "text"}, + {"name": "operator", "type": "text"}, + /* By using boolean columns stylesheet processing can be simplified */ + {"name": "shelter", "type": "boolean"}, + {"name": "bench", "type": "boolean"}, + {"name": "wheelchair", "type": "boolean"} + ] + }, + { + "name": "highways", + "type": "line", + "tagtransform": "multi.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "highway_ways", + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "name", "type": "text"}, + {"name": "name_en", "type": "text"}, + {"name": "name_fr", "type": "text"}, + {"name": "name_de", "type": "text"}, + {"name": "road_class", "type": "integer"}, + {"name": "road_type", "type": "text"} + ] + }, + { + "name": "buildings", + "type": "polygon", + "tagtransform": "multi.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "building_ways", + "tagtransform-relation-function": "building_rels", + "tagtransform-relation-member-function": "building_rel_members", + "tags": [ + {"name": "name", "type": "text"}, + {"name": "name_en", "type": "text"}, + {"name": "name_fr", "type": "text"}, + {"name": "name_de", "type": "text"}, + {"name": "building", "type": "text"}, + {"name": "shop", "type": "text"}, + {"name": "amenity", "type": "text"} + ] + } +] diff -Nru osm2pgsql-0.82.0/node-persistent-cache.c osm2pgsql-0.88.1/node-persistent-cache.c --- osm2pgsql-0.82.0/node-persistent-cache.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/node-persistent-cache.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,733 +0,0 @@ -#define _LARGEFILE64_SOURCE /* See feature_test_macrors(7) */ - -#include "config.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "osmtypes.h" -#include "output.h" -#include "node-persistent-cache.h" -#include "node-ram-cache.h" -#include "binarysearcharray.h" - -#ifdef __APPLE__ - #define lseek64 lseek -#else - #ifndef HAVE_LSEEK64 - #if SIZEOF_OFF_T == 8 - #define lseek64 lseek - #else - #error Flat nodes cache requires a 64 bit capable seek - #endif - #endif -#endif - -static int node_cache_fd; -static const char * node_cache_fname; -static int append_mode; - -struct persistentCacheHeader cacheHeader; -static struct ramNodeBlock writeNodeBlock; /* larger node block for more efficient initial sequential writing of node cache */ -static struct ramNodeBlock * readNodeBlockCache; -static struct binary_search_array * readNodeBlockCacheIdx; - - -static int scale; -static int cache_already_written = 0; - - -static void writeout_dirty_nodes(osmid_t id) -{ - int i; - - if (writeNodeBlock.dirty > 0) - { - if (lseek64(node_cache_fd, - (writeNodeBlock.block_offset << WRITE_NODE_BLOCK_SHIFT) - * sizeof(struct ramNode) - + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - - }; - if (write(node_cache_fd, writeNodeBlock.nodes, - WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - < WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - { - fprintf(stderr, "Failed to write out node cache: %s\n", - strerror(errno)); - exit_nicely(); - } - cacheHeader.max_initialised_id = ((writeNodeBlock.block_offset + 1) - << WRITE_NODE_BLOCK_SHIFT) - 1; - writeNodeBlock.used = 0; - writeNodeBlock.dirty = 0; - if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (write(node_cache_fd, &cacheHeader, - sizeof(struct persistentCacheHeader)) - != sizeof(struct persistentCacheHeader)) - { - fprintf(stderr, "Failed to update persistent cache header: %s\n", - strerror(errno)); - exit_nicely(); - } - if (fsync(node_cache_fd) < 0) { - fprintf(stderr, "Info: Node cache could not be guaranteeded to be made durable. fsync failed: %s\n", - strerror(errno)); - }; - } - if (id < 0) - { - for (i = 0; i < READ_NODE_CACHE_SIZE; i++) - { - if (readNodeBlockCache[i].dirty) - { - if (lseek64(node_cache_fd, - (readNodeBlockCache[i].block_offset - << READ_NODE_BLOCK_SHIFT) - * sizeof(struct ramNode) - + sizeof(struct persistentCacheHeader), - SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (write(node_cache_fd, readNodeBlockCache[i].nodes, - READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - < READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - { - fprintf(stderr, "Failed to write out node cache: %s\n", - strerror(errno)); - exit_nicely(); - } - } - readNodeBlockCache[i].dirty = 0; - } - } - -} - -static void ramNodes_clear(struct ramNode * nodes, int size) -{ - int i; - for (i = 0; i < size; i++) - { -#ifdef FIXED_POINT - nodes[i].lon = INT_MIN; - nodes[i].lat = INT_MIN; -#else - nodes[i].lon = NAN; - nodes[i].lat = NAN; -#endif - } -} - -/** - * Find the cache block with the lowest usage count for replacement - */ -static int persistent_cache_replace_block() -{ - int min_used = INT_MAX; - int block_id = -1; - int i; - - for (i = 0; i < READ_NODE_CACHE_SIZE; i++) - { - if (readNodeBlockCache[i].used < min_used) - { - min_used = readNodeBlockCache[i].used; - block_id = i; - } - } - if (min_used > 0) - { - for (i = 0; i < READ_NODE_CACHE_SIZE; i++) - { - if (readNodeBlockCache[i].used > 1) - { - readNodeBlockCache[i].used--; - } - } - } - return block_id; -} - -/** - * Find cache block number by block_offset - */ -static int persistent_cache_find_block(osmid_t block_offset) -{ - int idx = binary_search_get(readNodeBlockCacheIdx, block_offset); - return idx; -} - -/** - * Initialise the persistent cache with NaN values to identify which IDs are valid or not - */ -static void persistent_cache_expand_cache(osmid_t block_offset) -{ - osmid_t i; - struct ramNode * dummyNodes = malloc( - READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)); - if (!dummyNodes) { - fprintf(stderr, "Out of memory: Could not allocate node structure during cache expansion\n"); - exit_nicely(); - } - ramNodes_clear(dummyNodes, READ_NODE_BLOCK_SIZE); - /* Need to expand the persistent node cache */ - if (lseek64(node_cache_fd, - cacheHeader.max_initialised_id * sizeof(struct ramNode) - + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - for (i = cacheHeader.max_initialised_id >> READ_NODE_BLOCK_SHIFT; - i <= block_offset; i++) - { - if (write(node_cache_fd, dummyNodes, - READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - < READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - { - fprintf(stderr, "Failed to expand persistent node cache: %s\n", - strerror(errno)); - exit_nicely(); - } - } - cacheHeader.max_initialised_id = ((block_offset + 1) - << READ_NODE_BLOCK_SHIFT) - 1; - if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader)) - != sizeof(struct persistentCacheHeader)) - { - fprintf(stderr, "Failed to update persistent cache header: %s\n", - strerror(errno)); - exit_nicely(); - } - free(dummyNodes); - fsync(node_cache_fd); -} - - -static void persistent_cache_nodes_prefetch_async(osmid_t id) -{ -#ifdef HAVE_POSIX_FADVISE - osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; - - osmid_t block_id = persistent_cache_find_block(block_offset); - - if (block_id < 0) - { /* The needed block isn't in cache already, so initiate loading */ - writeout_dirty_nodes(id); - - /* Make sure the node cache is correctly initialised for the block that will be read */ - if (cacheHeader.max_initialised_id - < ((block_offset + 1) << READ_NODE_BLOCK_SHIFT)) - persistent_cache_expand_cache(block_offset); - - if (posix_fadvise(node_cache_fd, (block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(struct ramNode) - + sizeof(struct persistentCacheHeader), READ_NODE_BLOCK_SIZE * sizeof(struct ramNode), - POSIX_FADV_WILLNEED | POSIX_FADV_RANDOM) != 0) { - fprintf(stderr, "Info: async prefetch of node cache failed. This might reduce performance\n"); - }; - } -#endif -} - - -/** - * Load block offset in a synchronous way. - */ -static int persistent_cache_load_block(osmid_t block_offset) -{ - - int block_id = persistent_cache_replace_block(); - - if (readNodeBlockCache[block_id].dirty) - { - if (lseek64(node_cache_fd, - (readNodeBlockCache[block_id].block_offset - << READ_NODE_BLOCK_SHIFT) * sizeof(struct ramNode) - + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (write(node_cache_fd, readNodeBlockCache[block_id].nodes, - READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - < READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - { - fprintf(stderr, "Failed to write out node cache: %s\n", - strerror(errno)); - exit_nicely(); - } - readNodeBlockCache[block_id].dirty = 0; - } - - binary_search_remove(readNodeBlockCacheIdx, - readNodeBlockCache[block_id].block_offset); - ramNodes_clear(readNodeBlockCache[block_id].nodes, READ_NODE_BLOCK_SIZE); - readNodeBlockCache[block_id].block_offset = block_offset; - readNodeBlockCache[block_id].used = READ_NODE_CACHE_SIZE; - - /* Make sure the node cache is correctly initialised for the block that will be read */ - if (cacheHeader.max_initialised_id - < ((block_offset + 1) << READ_NODE_BLOCK_SHIFT)) - { - persistent_cache_expand_cache(block_offset); - } - - /* Read the block into cache */ - if (lseek64(node_cache_fd, - (block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(struct ramNode) - + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (read(node_cache_fd, readNodeBlockCache[block_id].nodes, - READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - != READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - { - fprintf(stderr, "Failed to read from node cache: %s\n", - strerror(errno)); - exit(1); - } - binary_search_add(readNodeBlockCacheIdx, - readNodeBlockCache[block_id].block_offset, block_id); - - return block_id; -} - -static void persisten_cache_nodes_set_create_writeout_block() -{ - if (write(node_cache_fd, writeNodeBlock.nodes, - WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - < WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode)) - { - fprintf(stderr, "Failed to write out node cache: %s\n", - strerror(errno)); - exit_nicely(); - } -#ifdef HAVE_SYNC_FILE_RANGE - /* writing out large files can cause trouble on some operating systems. - * For one, if to much dirty data is in RAM, the whole OS can stall until - * enough dirty data is written out which can take a while. It can also interfere - * with outher disk caching operations and might push things out to swap. By forcing the OS to - * immediately write out the data and blocking after a while, we ensure that no more - * than a couple of 10s of MB are dirty in RAM at a time. - * Secondly, the nodes are stored in an additional ram cache during import. Keeping the - * node cache file in buffer cache therefore duplicates the data wasting 16GB of ram. - * Therefore tell the OS not to cache the node-persistent-cache during initial import. - * */ - if (sync_file_range(node_cache_fd, writeNodeBlock.block_offset*WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode) + - sizeof(struct persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode), - SYNC_FILE_RANGE_WRITE) < 0) { - fprintf(stderr, "Info: Sync_file_range writeout has an issue. This shouldn't be anything to worry about.: %s\n", - strerror(errno)); - }; - - if (writeNodeBlock.block_offset > 16) { - if(sync_file_range(node_cache_fd, (writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode) + - sizeof(struct persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode), - SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER) < 0) { - fprintf(stderr, "Info: Sync_file_range block has an issue. This shouldn't be anything to worry about.: %s\n", - strerror(errno)); - - } -#ifdef HAVE_POSIX_FADVISE - if (posix_fadvise(node_cache_fd, (writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode) + - sizeof(struct persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode), POSIX_FADV_DONTNEED) !=0 ) { - fprintf(stderr, "Info: Posix_fadvise failed. This shouldn't be anything to worry about.: %s\n", - strerror(errno)); - }; -#endif - } -#endif -} - -static int persistent_cache_nodes_set_create(osmid_t id, double lat, double lon) -{ - osmid_t block_offset = id >> WRITE_NODE_BLOCK_SHIFT; - int i; - - if (cache_already_written) - return 0; - - if (writeNodeBlock.block_offset != block_offset) - { - if (writeNodeBlock.dirty) - { - persisten_cache_nodes_set_create_writeout_block(); - writeNodeBlock.used = 0; - writeNodeBlock.dirty = 0; - /* After writing out the node block, the file pointer is at the next block level */ - writeNodeBlock.block_offset++; - cacheHeader.max_initialised_id = (writeNodeBlock.block_offset - << WRITE_NODE_BLOCK_SHIFT) - 1; - } - if (writeNodeBlock.block_offset > block_offset) - { - fprintf(stderr, - "ERROR: Block_offset not in sequential order: %" PRIdOSMID "%" PRIdOSMID "\n", - writeNodeBlock.block_offset, block_offset); - exit_nicely(); - } - - /* We need to fill the intermediate node cache with node nodes to identify which nodes are valid */ - for (i = writeNodeBlock.block_offset; i < block_offset; i++) - { - ramNodes_clear(writeNodeBlock.nodes, WRITE_NODE_BLOCK_SIZE); - persisten_cache_nodes_set_create_writeout_block(); - } - - ramNodes_clear(writeNodeBlock.nodes, WRITE_NODE_BLOCK_SIZE); - writeNodeBlock.used = 0; - writeNodeBlock.block_offset = block_offset; - } -#ifdef FIXED_POINT - writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lat = DOUBLE_TO_FIX(lat); - writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lon = DOUBLE_TO_FIX(lon); -#else - writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lat = lat; - writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK].lon = lon; -#endif - writeNodeBlock.used++; - writeNodeBlock.dirty = 1; - - return 0; -} - -static int persistent_cache_nodes_set_append(osmid_t id, double lat, double lon) -{ - osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; - - int block_id = persistent_cache_find_block(block_offset); - - if (block_id < 0) - block_id = persistent_cache_load_block(block_offset); - -#ifdef FIXED_POINT - if (isnan(lat) && isnan(lon)) - { - readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat = - INT_MIN; - readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon = - INT_MIN; - } - else - { - readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat = - DOUBLE_TO_FIX(lat); - readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon = - DOUBLE_TO_FIX(lon); - } -#else - readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat = lat; - readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon = lon; -#endif - readNodeBlockCache[block_id].used++; - readNodeBlockCache[block_id].dirty = 1; - - return 1; -} - -int persistent_cache_nodes_set(osmid_t id, double lat, double lon) -{ - return append_mode ? - persistent_cache_nodes_set_append(id, lat, lon) : - persistent_cache_nodes_set_create(id, lat, lon); -} - -int persistent_cache_nodes_get(struct osmNode *out, osmid_t id) -{ - osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; - - osmid_t block_id = persistent_cache_find_block(block_offset); - - if (block_id < 0) - { - writeout_dirty_nodes(id); - block_id = persistent_cache_load_block(block_offset); - } - - readNodeBlockCache[block_id].used++; - -#ifdef FIXED_POINT - if ((readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat - == INT_MIN) - && (readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon - == INT_MIN)) - { - return 1; - } - else - { - out->lat = - FIX_TO_DOUBLE(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat); - out->lon = - FIX_TO_DOUBLE(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon); - return 0; - } -#else - if ((isnan(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat)) && - (isnan(readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon))) - { - return 1; - } - else - { - out->lat = readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat; - out->lon = readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon; - return 0; - } -#endif - - return 0; -} - -int persistent_cache_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, - int nd_count) -{ - int count = 0; - int i; - for (i = 0; i < nd_count; i++) - { - /* Check cache first */ - if (ram_cache_nodes_get(&nodes[i], ndids[i]) == 0) - { - count++; - } - else - { - nodes[i].lat = NAN; - nodes[i].lon = NAN; - } - } - if (count == nd_count) - return count; - - for (i = 0; i < nd_count; i++) - { - /* In order to have a higher OS level I/O queue depth - issue posix_fadvise(WILLNEED) requests for all I/O */ - if (isnan(nodes[i].lat) && isnan(nodes[i].lon)) - persistent_cache_nodes_prefetch_async(ndids[i]); - } - for (i = 0; i < nd_count; i++) - { - if ((isnan(nodes[i].lat) && isnan(nodes[i].lon)) - && (persistent_cache_nodes_get(&(nodes[i]), ndids[i]) == 0)) - count++; - } - - if (count < nd_count) - { - int j = 0; - for (i = 0; i < nd_count; i++) - { - if (!isnan(nodes[i].lat)) - { - nodes[j].lat = nodes[i].lat; - nodes[j].lon = nodes[i].lon; - j++; - } - } - for (i = count; i < nd_count; i++) - { - nodes[i].lat = NAN; - nodes[i].lon = NAN; - } - } - - return count; -} - -void init_node_persistent_cache(const struct output_options *options, int append) -{ - int i; - scale = options->scale; - append_mode = append; - node_cache_fname = options->flat_node_file; - fprintf(stderr, "Mid: loading persistent node cache from %s\n", - node_cache_fname); - - readNodeBlockCacheIdx = init_search_array(READ_NODE_CACHE_SIZE); - - /* Setup the file for the node position cache */ - if (append_mode) - { - node_cache_fd = open(node_cache_fname, O_RDWR, S_IRUSR | S_IWUSR); - if (node_cache_fd < 0) - { - fprintf(stderr, "Failed to open node cache file: %s\n", - strerror(errno)); - exit_nicely(); - } - } - else - { - if (cache_already_written) - { - node_cache_fd = open(node_cache_fname, O_RDWR, S_IRUSR | S_IWUSR); - } - else - { - node_cache_fd = open(node_cache_fname, O_RDWR | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR); - } - - if (node_cache_fd < 0) - { - fprintf(stderr, "Failed to create node cache file: %s\n", - strerror(errno)); - exit_nicely(); - } - if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (cache_already_written == 0) - { - - #ifdef HAVE_POSIX_FALLOCATE - if (posix_fallocate(node_cache_fd, 0, - sizeof(struct ramNode) * MAXIMUM_INITIAL_ID) != 0) - { - fprintf(stderr, - "Failed to allocate space for node cache file: %s\n", - strerror(errno)); - close(node_cache_fd); - exit_nicely(); - } - fprintf(stderr, "Allocated space for persistent node cache file\n"); - #endif - writeNodeBlock.nodes = malloc( - WRITE_NODE_BLOCK_SIZE * sizeof(struct ramNode)); - if (!writeNodeBlock.nodes) { - fprintf(stderr, "Out of memory: Failed to allocate node writeout buffer\n"); - exit_nicely(); - } - ramNodes_clear(writeNodeBlock.nodes, WRITE_NODE_BLOCK_SIZE); - writeNodeBlock.block_offset = 0; - writeNodeBlock.used = 0; - writeNodeBlock.dirty = 0; - cacheHeader.format_version = PERSISTENT_CACHE_FORMAT_VERSION; - cacheHeader.id_size = sizeof(osmid_t); - cacheHeader.max_initialised_id = 0; - if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (write(node_cache_fd, &cacheHeader, - sizeof(struct persistentCacheHeader)) - != sizeof(struct persistentCacheHeader)) - { - fprintf(stderr, "Failed to write persistent cache header: %s\n", - strerror(errno)); - exit_nicely(); - } - } - - } - if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (read(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader)) - != sizeof(struct persistentCacheHeader)) - { - fprintf(stderr, "Failed to read persistent cache header: %s\n", - strerror(errno)); - exit_nicely(); - } - if (cacheHeader.format_version != PERSISTENT_CACHE_FORMAT_VERSION) - { - fprintf(stderr, "Persistent cache header is wrong version\n"); - exit_nicely(); - } - - if (cacheHeader.id_size != sizeof(osmid_t)) - { - fprintf(stderr, "Persistent cache header is wrong id type\n"); - exit_nicely(); - } - - fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id); - - readNodeBlockCache = malloc( - READ_NODE_CACHE_SIZE * sizeof(struct ramNodeBlock)); - if (!readNodeBlockCache) { - fprintf(stderr, "Out of memory: Failed to allocate node read cache\n"); - exit_nicely(); - } - for (i = 0; i < READ_NODE_CACHE_SIZE; i++) - { - readNodeBlockCache[i].nodes = malloc( - READ_NODE_BLOCK_SIZE * sizeof(struct ramNode)); - if (!readNodeBlockCache[i].nodes) { - fprintf(stderr, "Out of memory: Failed to allocate node read cache\n"); - exit_nicely(); - } - readNodeBlockCache[i].block_offset = -1; - readNodeBlockCache[i].used = 0; - readNodeBlockCache[i].dirty = 0; - } -} - -void shutdown_node_persistent_cache() -{ - int i; - writeout_dirty_nodes(-1); - - if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { - fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", - strerror(errno)); - exit_nicely(); - }; - if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader)) - != sizeof(struct persistentCacheHeader)) - { - fprintf(stderr, "Failed to update persistent cache header: %s\n", - strerror(errno)); - exit_nicely(); - } - fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id); - - fsync(node_cache_fd); - - if (close(node_cache_fd) != 0) - { - fprintf(stderr, "Failed to close node cache file: %s\n", - strerror(errno)); - } - - for (i = 0; i < READ_NODE_CACHE_SIZE; i++) - { - free(readNodeBlockCache[i].nodes); - } - shutdown_search_array(&readNodeBlockCacheIdx); - free(readNodeBlockCache); - readNodeBlockCache = NULL; -} diff -Nru osm2pgsql-0.82.0/node-persistent-cache.cpp osm2pgsql-0.88.1/node-persistent-cache.cpp --- osm2pgsql-0.82.0/node-persistent-cache.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/node-persistent-cache.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,658 @@ +#define _LARGEFILE64_SOURCE /* See feature_test_macrors(7) */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "output.hpp" +#include "options.hpp" +#include "node-persistent-cache.hpp" +#include "util.hpp" + +#include +#include + +#ifdef _WIN32 + #include "win_fsync.h" + #define lseek64 _lseeki64 + #ifndef S_IRUSR + #define S_IRUSR S_IREAD + #endif + #ifndef S_IWUSR + #define S_IWUSR S_IWRITE + #endif +#else + #ifdef __APPLE__ + #define lseek64 lseek + #else + #ifndef HAVE_LSEEK64 + #if SIZEOF_OFF_T == 8 + #define lseek64 lseek + #else + #error Flat nodes cache requires a 64 bit capable seek + #endif + #endif + #endif +#endif + +void node_persistent_cache::writeout_dirty_nodes() +{ + for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) + { + if (readNodeBlockCache[i].dirty()) + { + if (lseek64(node_cache_fd, + ((osmid_t) readNodeBlockCache[i].block_offset + << READ_NODE_BLOCK_SHIFT) + * sizeof(ramNode) + + sizeof(persistentCacheHeader), + SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (write(node_cache_fd, readNodeBlockCache[i].nodes, + READ_NODE_BLOCK_SIZE * sizeof(ramNode)) + < ssize_t(READ_NODE_BLOCK_SIZE * sizeof(ramNode))) + { + fprintf(stderr, "Failed to write out node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + } + } + readNodeBlockCache[i].reset_used(); + } +} + + +/** + * Find the cache block with the lowest usage count for replacement + */ +int node_persistent_cache::replace_block() +{ + int min_used = INT_MAX; + int block_id = -1; + + for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) + { + if (readNodeBlockCache[i].used() < min_used) + { + min_used = readNodeBlockCache[i].used(); + block_id = i; + } + } + if (min_used > 0) + { + for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) + { + if (readNodeBlockCache[i].used() > 1) + { + readNodeBlockCache[i].dec_used(); + } + } + } + return block_id; +} + +/** + * Find cache block number by block_offset + */ +int node_persistent_cache::find_block(osmid_t block_offset) +{ + cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(), + readNodeBlockCacheIdx.end(), + block_offset); + if (it != readNodeBlockCacheIdx.end() && it->key == block_offset) + return it->value; + + return -1; +} + +void node_persistent_cache::remove_from_cache_idx(osmid_t block_offset) +{ + cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(), + readNodeBlockCacheIdx.end(), + block_offset); + + if (it == readNodeBlockCacheIdx.end() || it->key != block_offset) + return; + + readNodeBlockCacheIdx.erase(it); +} + +void node_persistent_cache::add_to_cache_idx(cache_index_entry const &entry) +{ + cache_index::iterator it = std::lower_bound(readNodeBlockCacheIdx.begin(), + readNodeBlockCacheIdx.end(), + entry); + readNodeBlockCacheIdx.insert(it, entry); +} + +// A cache block with invalid nodes, just for writing out empty cache blocks +static const ramNode nullNodes[READ_NODE_BLOCK_SIZE]; +/** + * Initialise the persistent cache with NaN values to identify which IDs are valid or not + */ +void node_persistent_cache::expand_cache(osmid_t block_offset) +{ + /* Need to expand the persistent node cache */ + if (lseek64(node_cache_fd, + cacheHeader.max_initialised_id * sizeof(ramNode) + + sizeof(persistentCacheHeader), SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + for (osmid_t i = cacheHeader.max_initialised_id >> READ_NODE_BLOCK_SHIFT; + i <= block_offset; i++) + { + if (write(node_cache_fd, nullNodes, sizeof(nullNodes)) + < ssize_t(sizeof(nullNodes))) + { + fprintf(stderr, "Failed to expand persistent node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + } + } + cacheHeader.max_initialised_id = ((block_offset + 1) + << READ_NODE_BLOCK_SHIFT) - 1; + if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader)) + != ssize_t(sizeof(struct persistentCacheHeader))) + { + fprintf(stderr, "Failed to update persistent cache header: %s\n", + strerror(errno)); + util::exit_nicely(); + } + fsync(node_cache_fd); +} + + +void node_persistent_cache::nodes_prefetch_async(osmid_t id) +{ +#ifdef HAVE_POSIX_FADVISE + osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; + + const int block_id = find_block(block_offset); + + if (block_id < 0) { + // The needed block isn't in cache already, so initiate loading + if (cacheHeader.max_initialised_id < id) { + fprintf(stderr, "Warning: reading node outside node cache. (%lu vs. %lu)\n", + cacheHeader.max_initialised_id, id); + return; + } + + if (posix_fadvise(node_cache_fd, (block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode) + + sizeof(persistentCacheHeader), READ_NODE_BLOCK_SIZE * sizeof(ramNode), + POSIX_FADV_WILLNEED | POSIX_FADV_RANDOM) != 0) { + fprintf(stderr, "Info: async prefetch of node cache failed. This might reduce performance\n"); + }; + } +#endif +} + + +/** + * Load block offset in a synchronous way. + */ +int node_persistent_cache::load_block(osmid_t block_offset) +{ + const int block_id = replace_block(); + + if (readNodeBlockCache[block_id].dirty()) + { + if (lseek64(node_cache_fd, + ((osmid_t) readNodeBlockCache[block_id].block_offset + << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode) + + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (write(node_cache_fd, readNodeBlockCache[block_id].nodes, + READ_NODE_BLOCK_SIZE * sizeof(ramNode)) + < ssize_t(READ_NODE_BLOCK_SIZE * sizeof(ramNode))) + { + fprintf(stderr, "Failed to write out node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + } + readNodeBlockCache[block_id].reset_used(); + } + + if (readNodeBlockCache[block_id].nodes) { + remove_from_cache_idx((osmid_t) readNodeBlockCache[block_id].block_offset); + new(readNodeBlockCache[block_id].nodes) ramNode[READ_NODE_BLOCK_SIZE]; + } else { + readNodeBlockCache[block_id].nodes = new ramNode[READ_NODE_BLOCK_SIZE]; + if (!readNodeBlockCache[block_id].nodes) { + fprintf(stderr, "Out of memory: Failed to allocate node read cache\n"); + util::exit_nicely(); + } + } + readNodeBlockCache[block_id].block_offset = block_offset; + readNodeBlockCache[block_id].set_used(READ_NODE_CACHE_SIZE); + + /* Make sure the node cache is correctly initialised for the block that will be read */ + if (cacheHeader.max_initialised_id + < ((block_offset + 1) << READ_NODE_BLOCK_SHIFT)) + { + expand_cache(block_offset); + } + + /* Read the block into cache */ + if (lseek64(node_cache_fd, + (block_offset << READ_NODE_BLOCK_SHIFT) * sizeof(ramNode) + + sizeof(struct persistentCacheHeader), SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (read(node_cache_fd, readNodeBlockCache[block_id].nodes, + READ_NODE_BLOCK_SIZE * sizeof(ramNode)) + != READ_NODE_BLOCK_SIZE * sizeof(ramNode)) + { + fprintf(stderr, "Failed to read from node cache: %s\n", + strerror(errno)); + exit(1); + } + add_to_cache_idx(cache_index_entry(block_offset, block_id)); + + return block_id; +} + +void node_persistent_cache::nodes_set_create_writeout_block() +{ + if (write(node_cache_fd, writeNodeBlock.nodes, + WRITE_NODE_BLOCK_SIZE * sizeof(ramNode)) + < ssize_t(WRITE_NODE_BLOCK_SIZE * sizeof(ramNode))) + { + fprintf(stderr, "Failed to write out node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + } +#ifdef HAVE_SYNC_FILE_RANGE + /* writing out large files can cause trouble on some operating systems. + * For one, if to much dirty data is in RAM, the whole OS can stall until + * enough dirty data is written out which can take a while. It can also interfere + * with other disk caching operations and might push things out to swap. By forcing the OS to + * immediately write out the data and blocking after a while, we ensure that no more + * than a couple of 10s of MB are dirty in RAM at a time. + * Secondly, the nodes are stored in an additional ram cache during import. Keeping the + * node cache file in buffer cache therefore duplicates the data wasting 16GB of ram. + * Therefore tell the OS not to cache the node-persistent-cache during initial import. + * */ + if (sync_file_range(node_cache_fd, (osmid_t) writeNodeBlock.block_offset*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + + sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode), + SYNC_FILE_RANGE_WRITE) < 0) { + fprintf(stderr, "Info: Sync_file_range writeout has an issue. This shouldn't be anything to worry about.: %s\n", + strerror(errno)); + }; + + if (writeNodeBlock.block_offset > 16) { + if(sync_file_range(node_cache_fd, ((osmid_t) writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + + sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode), + SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER) < 0) { + fprintf(stderr, "Info: Sync_file_range block has an issue. This shouldn't be anything to worry about.: %s\n", + strerror(errno)); + + } +#ifdef HAVE_POSIX_FADVISE + if (posix_fadvise(node_cache_fd, ((osmid_t) writeNodeBlock.block_offset - 16)*WRITE_NODE_BLOCK_SIZE * sizeof(ramNode) + + sizeof(persistentCacheHeader), WRITE_NODE_BLOCK_SIZE * sizeof(ramNode), POSIX_FADV_DONTNEED) !=0 ) { + fprintf(stderr, "Info: Posix_fadvise failed. This shouldn't be anything to worry about.: %s\n", + strerror(errno)); + }; +#endif + } +#endif +} + +int node_persistent_cache::set_create(osmid_t id, double lat, double lon) +{ + assert(!append_mode); + assert(!read_mode); + + int32_t block_offset = id >> WRITE_NODE_BLOCK_SHIFT; + + if (writeNodeBlock.block_offset != block_offset) + { + if (writeNodeBlock.dirty()) + { + nodes_set_create_writeout_block(); + /* After writing out the node block, the file pointer is at the next block level */ + writeNodeBlock.block_offset++; + cacheHeader.max_initialised_id = ((osmid_t) writeNodeBlock.block_offset + << WRITE_NODE_BLOCK_SHIFT) - 1; + } + if (writeNodeBlock.block_offset > block_offset) + { + fprintf(stderr, + "ERROR: Block_offset not in sequential order: %d %d\n", + writeNodeBlock.block_offset, block_offset); + util::exit_nicely(); + } + + new(writeNodeBlock.nodes) ramNode[WRITE_NODE_BLOCK_SIZE]; + + /* We need to fill the intermediate node cache with node nodes to identify which nodes are valid */ + while (writeNodeBlock.block_offset < block_offset) + { + nodes_set_create_writeout_block(); + writeNodeBlock.block_offset++; + } + + } + + writeNodeBlock.nodes[id & WRITE_NODE_BLOCK_MASK] = ramNode(lon, lat); + writeNodeBlock.set_dirty(); + + return 0; +} + +int node_persistent_cache::set_append(osmid_t id, double lat, double lon) +{ + assert(!read_mode); + + osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; + + int block_id = find_block(block_offset); + + if (block_id < 0) + block_id = load_block(block_offset); + + if (std::isnan(lat) && std::isnan(lon)) { + readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK] = ramNode(); + } else { + readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK] = ramNode(lon, lat); + } + readNodeBlockCache[block_id].inc_used(); + readNodeBlockCache[block_id].set_dirty(); + + return 1; +} + +int node_persistent_cache::set(osmid_t id, double lat, double lon) +{ + return append_mode ? + set_append(id, lat, lon) : + set_create(id, lat, lon); +} + +int node_persistent_cache::get(osmNode *out, osmid_t id) +{ + set_read_mode(); + + osmid_t block_offset = id >> READ_NODE_BLOCK_SHIFT; + + int block_id = find_block(block_offset); + + if (block_id < 0) + { + block_id = load_block(block_offset); + } + + readNodeBlockCache[block_id].inc_used(); + + if (!readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].is_valid()) + return 1; + + out->lat = readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lat(); + out->lon = readNodeBlockCache[block_id].nodes[id & READ_NODE_BLOCK_MASK].lon(); + + return 0; +} + +int node_persistent_cache::get_list(nodelist_t &out, const idlist_t nds) +{ + set_read_mode(); + + out.assign(nds.size(), osmNode()); + + bool need_fetch = false; + for (size_t i = 0; i < nds.size(); ++i) { + /* Check cache first */ + if (ram_cache->get(&out[i], nds[i]) != 0) { + /* In order to have a higher OS level I/O queue depth + issue posix_fadvise(WILLNEED) requests for all I/O */ + nodes_prefetch_async(nds[i]); + need_fetch = true; + } + } + if (!need_fetch) + return out.size(); + + size_t wrtidx = 0; + for (size_t i = 0; i < nds.size(); i++) { + if (std::isnan(out[i].lat) && std::isnan(out[i].lon)) { + if (get(&(out[wrtidx]), nds[i]) == 0) + wrtidx++; + } else { + if (wrtidx < i) + out[wrtidx] = out[i]; + wrtidx++; + } + } + + out.resize(wrtidx); + + return wrtidx; +} + +void node_persistent_cache::set_read_mode() +{ + if (read_mode) + return; + + if (writeNodeBlock.dirty()) { + assert(!append_mode); + nodes_set_create_writeout_block(); + writeNodeBlock.reset_used(); + writeNodeBlock.block_offset++; + cacheHeader.max_initialised_id = ((osmid_t) writeNodeBlock.block_offset + << WRITE_NODE_BLOCK_SHIFT) - 1; + + /* write out the header */ + if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (write(node_cache_fd, &cacheHeader, sizeof(persistentCacheHeader)) + != sizeof(persistentCacheHeader)) { + fprintf(stderr, "Failed to update persistent cache header: %s\n", + strerror(errno)); + util::exit_nicely(); + } + } + + read_mode = true; +} + +node_persistent_cache::node_persistent_cache(const options_t *options, int append, + bool ro, boost::shared_ptr ptr) + : node_cache_fd(0), node_cache_fname(NULL), append_mode(0), cacheHeader(), + writeNodeBlock(), readNodeBlockCache(NULL), read_mode(ro), ram_cache(ptr) +{ + append_mode = append; + if (options->flat_node_file) { + node_cache_fname = options->flat_node_file->c_str(); + } else { + throw std::runtime_error("Unable to set up persistent cache: the name " + "of the flat node file was not set."); + } + fprintf(stderr, "Mid: loading persistent node cache from %s\n", + node_cache_fname); + + readNodeBlockCacheIdx.reserve(READ_NODE_CACHE_SIZE); + + /* Setup the file for the node position cache */ + if (append_mode) + { + node_cache_fd = open(node_cache_fname, O_RDWR, S_IRUSR | S_IWUSR); + if (node_cache_fd < 0) + { + fprintf(stderr, "Failed to open node cache file: %s\n", + strerror(errno)); + util::exit_nicely(); + } + } + else + { + if (read_mode) + { + node_cache_fd = open(node_cache_fname, O_RDWR, S_IRUSR | S_IWUSR); + } + else + { + node_cache_fd = open(node_cache_fname, O_RDWR | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR); + } + + if (node_cache_fd < 0) + { + fprintf(stderr, "Failed to create node cache file: %s\n", + strerror(errno)); + util::exit_nicely(); + } + if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + + writeNodeBlock.block_offset = 0; + + if (!read_mode) + { + int err; + #ifdef HAVE_POSIX_FALLOCATE + if ((err = posix_fallocate(node_cache_fd, 0, + sizeof(ramNode) * MAXIMUM_INITIAL_ID)) != 0) + { + if (err == ENOSPC) { + fprintf(stderr, "Failed to allocate space for node cache file: No space on disk\n"); + } else if (err == EFBIG) { + fprintf(stderr, "Failed to allocate space for node cache file: File is too big\n"); + } else { + fprintf(stderr, "Failed to allocate space for node cache file: Internal error %i\n", err); + } + + close(node_cache_fd); + util::exit_nicely(); + } + fprintf(stderr, "Allocated space for persistent node cache file\n"); + #endif + writeNodeBlock.nodes = new ramNode[WRITE_NODE_BLOCK_SIZE]; + if (!writeNodeBlock.nodes) { + fprintf(stderr, "Out of memory: Failed to allocate node writeout buffer\n"); + util::exit_nicely(); + } + cacheHeader.format_version = PERSISTENT_CACHE_FORMAT_VERSION; + cacheHeader.id_size = sizeof(osmid_t); + cacheHeader.max_initialised_id = 0; + if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (write(node_cache_fd, &cacheHeader, + sizeof(struct persistentCacheHeader)) + != sizeof(struct persistentCacheHeader)) + { + fprintf(stderr, "Failed to write persistent cache header: %s\n", + strerror(errno)); + util::exit_nicely(); + } + } + + } + if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (read(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader)) + != sizeof(struct persistentCacheHeader)) + { + fprintf(stderr, "Failed to read persistent cache header: %s\n", + strerror(errno)); + util::exit_nicely(); + } + if (cacheHeader.format_version != PERSISTENT_CACHE_FORMAT_VERSION) + { + fprintf(stderr, "Persistent cache header is wrong version\n"); + util::exit_nicely(); + } + + if (cacheHeader.id_size != sizeof(osmid_t)) + { + fprintf(stderr, "Persistent cache header is wrong id type\n"); + util::exit_nicely(); + } + + fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id); + + readNodeBlockCache = new ramNodeBlock[READ_NODE_CACHE_SIZE]; + if (!readNodeBlockCache) { + fprintf(stderr, "Out of memory: Failed to allocate node read cache\n"); + util::exit_nicely(); + } +} + +node_persistent_cache::~node_persistent_cache() +{ + if (writeNodeBlock.dirty()) + nodes_set_create_writeout_block(); + + writeout_dirty_nodes(); + + if (writeNodeBlock.nodes) + delete[] writeNodeBlock.nodes; + + if (lseek64(node_cache_fd, 0, SEEK_SET) < 0) { + fprintf(stderr, "Failed to seek to correct position in node cache: %s\n", + strerror(errno)); + util::exit_nicely(); + }; + if (write(node_cache_fd, &cacheHeader, sizeof(struct persistentCacheHeader)) + != sizeof(struct persistentCacheHeader)) + { + fprintf(stderr, "Failed to update persistent cache header: %s\n", + strerror(errno)); + util::exit_nicely(); + } + fprintf(stderr,"Maximum node in persistent node cache: %" PRIdOSMID "\n", cacheHeader.max_initialised_id); + + fsync(node_cache_fd); + + if (close(node_cache_fd) != 0) + { + fprintf(stderr, "Failed to close node cache file: %s\n", + strerror(errno)); + } + + if (readNodeBlockCache) { + for (int i = 0; i < READ_NODE_CACHE_SIZE; i++) + { + if (readNodeBlockCache[i].nodes) + delete[] readNodeBlockCache[i].nodes; + } + delete[] readNodeBlockCache; + } +} diff -Nru osm2pgsql-0.82.0/node-persistent-cache.h osm2pgsql-0.88.1/node-persistent-cache.h --- osm2pgsql-0.82.0/node-persistent-cache.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/node-persistent-cache.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,24 +0,0 @@ -#define MAXIMUM_INITIAL_ID (1L << 31) - -#define READ_NODE_CACHE_SIZE 10000 -#define READ_NODE_BLOCK_SHIFT 10l -#define READ_NODE_BLOCK_SIZE (1l << READ_NODE_BLOCK_SHIFT) -#define READ_NODE_BLOCK_MASK 0x03FFl - -#define WRITE_NODE_BLOCK_SHIFT 20l -#define WRITE_NODE_BLOCK_SIZE (1l << WRITE_NODE_BLOCK_SHIFT) -#define WRITE_NODE_BLOCK_MASK 0x0FFFFFl - -#define PERSISTENT_CACHE_FORMAT_VERSION 1 - -struct persistentCacheHeader { - int format_version; - int id_size; - osmid_t max_initialised_id; -}; - -int persistent_cache_nodes_set(osmid_t id, double lat, double lon); -int persistent_cache_nodes_get(struct osmNode *out, osmid_t id); -int persistent_cache_nodes_get_list(struct osmNode *nodes, osmid_t *ndids, int nd_count); -void init_node_persistent_cache(const struct output_options *options, const int append); -void shutdown_node_persistent_cache(); diff -Nru osm2pgsql-0.82.0/node-persistent-cache.hpp osm2pgsql-0.88.1/node-persistent-cache.hpp --- osm2pgsql-0.82.0/node-persistent-cache.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/node-persistent-cache.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,95 @@ +#ifndef NODE_PERSISTENT_CACHE_H +#define NODE_PERSISTENT_CACHE_H + +#include "osmtypes.hpp" +#include "node-ram-cache.hpp" +#include + +#include + +#define MAXIMUM_INITIAL_ID 2600000000 + +#define READ_NODE_CACHE_SIZE 10000 +#define READ_NODE_BLOCK_SHIFT 10l +#define READ_NODE_BLOCK_SIZE (1l << READ_NODE_BLOCK_SHIFT) +#define READ_NODE_BLOCK_MASK 0x03FFl + +#define WRITE_NODE_BLOCK_SHIFT 20l +#define WRITE_NODE_BLOCK_SIZE (1l << WRITE_NODE_BLOCK_SHIFT) +#define WRITE_NODE_BLOCK_MASK 0x0FFFFFl + +#define PERSISTENT_CACHE_FORMAT_VERSION 1 + +struct persistentCacheHeader { + int format_version; + int id_size; + osmid_t max_initialised_id; +}; + +struct cache_index_entry { + osmid_t key; + int value; + + cache_index_entry(osmid_t k, int v) : key(k), value(v) {} + cache_index_entry() {} +}; + +inline bool operator<(cache_index_entry const &a, cache_index_entry const &b) +{ + return a.key < b.key; +} + +inline bool operator<(cache_index_entry const &a, osmid_t b) +{ + return a.key < b; +} + +inline bool operator<(osmid_t a, cache_index_entry const &b) +{ + return a < b.key; +} + +struct node_persistent_cache : public boost::noncopyable +{ + node_persistent_cache(const struct options_t *options, int append, + bool ro, boost::shared_ptr ptr); + ~node_persistent_cache(); + + int set(osmid_t id, double lat, double lon); + int get(osmNode *out, osmid_t id); + int get_list(nodelist_t &out, const idlist_t nds); + +private: + + int set_append(osmid_t id, double lat, double lon); + int set_create(osmid_t id, double lat, double lon); + + void writeout_dirty_nodes(); + int replace_block(); + int find_block(osmid_t block_offset); + void expand_cache(osmid_t block_offset); + void nodes_prefetch_async(osmid_t id); + int load_block(osmid_t block_offset); + void nodes_set_create_writeout_block(); + + void remove_from_cache_idx(osmid_t block_offset); + void add_to_cache_idx(cache_index_entry const &entry); + void set_read_mode(); + + int node_cache_fd; + const char * node_cache_fname; + int append_mode; + + persistentCacheHeader cacheHeader; + ramNodeBlock writeNodeBlock; /* larger node block for more efficient initial sequential writing of node cache */ + ramNodeBlock * readNodeBlockCache; + + typedef std::vector cache_index; + cache_index readNodeBlockCacheIdx; + + bool read_mode; + + boost::shared_ptr ram_cache; +}; + +#endif diff -Nru osm2pgsql-0.82.0/node-persistent-cache-reader.c osm2pgsql-0.88.1/node-persistent-cache-reader.c --- osm2pgsql-0.82.0/node-persistent-cache-reader.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/node-persistent-cache-reader.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,155 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "osmtypes.h" -#include "output.h" -#include "node-persistent-cache.h" -#include "node-ram-cache.h" -#include "binarysearcharray.h" - -void exit_nicely() -{ - fprintf(stderr, "Error occurred, cleaning up\n"); - exit(1); -} - -void test_get_node_list(int itterations, int max_size, int process_number) { - int i, j, node_cnt, node_cnt_total; - struct osmNode *nodes; - struct timeval start, stop; - struct timeval start_overall, stop_overall; - osmid_t *osmids; - - node_cnt_total = 0; - gettimeofday(&start_overall, NULL); - for (i = 0; i < itterations; i++) { - node_cnt = random() % max_size; - node_cnt_total += node_cnt; - - printf("Process %i: Getting %i nodes....\n", process_number, node_cnt); - nodes = malloc(sizeof(struct osmNode) * node_cnt); - osmids = malloc(sizeof(osmid_t) * node_cnt); - for (j = 0; j < node_cnt; j++) { - osmids[j] = random() % (1 << 31); - } - gettimeofday(&start, NULL); - persistent_cache_nodes_get_list(nodes,osmids,node_cnt); - gettimeofday(&stop, NULL); - double duration = ((stop.tv_sec - start.tv_sec)*1000000.0 + (stop.tv_usec - start.tv_usec))/1000000.0; - printf("Process %i: Got nodes in %f at a rate of %f/s\n", process_number, duration, node_cnt / duration); - free(nodes); - free(osmids); - } - gettimeofday(&stop_overall, NULL); - double duration = ((stop_overall.tv_sec - start_overall.tv_sec)*1000000.0 + (stop_overall.tv_usec - start_overall.tv_usec))/1000000.0; - printf("Process %i: Got a total of nodes in %f at a rate of %f/s\n", process_number, duration, node_cnt_total / duration); -} - -int main(int argc, char *argv[]) { - int i,p; - struct output_options options; - struct osmNode node; - struct osmNode *nodes; - struct timeval start; - osmid_t *osmids; - int node_cnt; - options.append = 1; - options.scale = 100; - options.flat_node_cache_enabled = 1; - options.flat_node_file = argv[1]; - init_node_ram_cache(0,10,100); - - - if (argc > 3) { - init_node_persistent_cache(&options, 1); - node_cnt = argc - 2; - nodes = malloc(sizeof(struct osmNode) * node_cnt); - osmids = malloc(sizeof(osmid_t) * node_cnt); - for (i = 0; i < node_cnt; i++) { - osmids[i] = atoi(argv[2 + i]); - } - persistent_cache_nodes_get_list(nodes,osmids,node_cnt); - for (i = 0; i < node_cnt; i++) { - printf("lat: %f / lon: %f\n", nodes[i].lat, nodes[i].lon); - } - } else if (argc == 2) { - char * state = malloc(sizeof(char)* 128); - gettimeofday(&start, NULL); - initstate(start.tv_usec, state, 8); - setstate(state); - - printf("Testing mode\n"); - init_node_persistent_cache(&options, 1); - test_get_node_list(10, 200, 0); - shutdown_node_persistent_cache(); -#ifdef HAVE_FORK - printf("Testing using multiple processes\n"); - int noProcs = 4; - int pid; - for (p = 1; p < noProcs; p++) { - pid=fork(); - if (pid==0) { - break; - } - if (pid==-1) { - fprintf(stderr,"WARNING: Failed to fork helper processes. Falling back to only using %i \n", p); - exit(1); - } - } - gettimeofday(&start, NULL); - initstate(start.tv_usec, state, 8); - setstate(state); - init_node_persistent_cache(&options, 1); - test_get_node_list(10,200,p); - - if (pid == 0) { - shutdown_node_persistent_cache(); - fprintf(stderr,"Exiting process %i\n", p); - exit(0); - } else { - for (p = 0; p < noProcs; p++) wait(NULL); - } - free(state); - fprintf(stderr, "\nAll child processes exited\n"); -#endif - } else { - init_node_persistent_cache(&options, 1); - if (strstr(argv[2],",") == NULL) { - persistent_cache_nodes_get(&node, atoi(argv[2])); - printf("lat: %f / lon: %f\n", node.lat, node.lon); - } else { - char * node_list = malloc(sizeof(char) * (strlen(argv[2]) + 1)); - strcpy(node_list,argv[2]); - node_cnt = 1; - strtok(node_list,","); - while (strtok(NULL,",") != NULL) node_cnt++; - printf("Processing %i nodes\n", node_cnt); - nodes = malloc(sizeof(struct osmNode) * node_cnt); - osmids = malloc(sizeof(osmid_t) * node_cnt); - strcpy(node_list,argv[2]); - osmids[0] = atoi(strtok(node_list,",")); - for (i = 1; i < node_cnt; i++) { - char * tmp = strtok(NULL,","); - osmids[i] = atoi(tmp); - } - persistent_cache_nodes_get_list(nodes,osmids,node_cnt); - for (i = 0; i < node_cnt; i++) { - printf("lat: %f / lon: %f\n", nodes[i].lat, nodes[i].lon); - } - } - } - - - shutdown_node_persistent_cache(); - return 0; -} diff -Nru osm2pgsql-0.82.0/node-persistent-cache-reader.cpp osm2pgsql-0.88.1/node-persistent-cache-reader.cpp --- osm2pgsql-0.82.0/node-persistent-cache-reader.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/node-persistent-cache-reader.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,140 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "options.hpp" +#include "node-persistent-cache.hpp" +#include "node-ram-cache.hpp" + +void test_get_node_list(boost::shared_ptr cache, + int itterations, int max_size, int process_number) { + int i, j, node_cnt, node_cnt_total; + nodelist_t nodes; + struct timeval start, stop; + struct timeval start_overall, stop_overall; + idlist_t osmids; + + node_cnt_total = 0; + gettimeofday(&start_overall, NULL); + for (i = 0; i < itterations; i++) { + node_cnt = random() % max_size; + node_cnt_total += node_cnt; + + printf("Process %i: Getting %i nodes....\n", process_number, node_cnt); + for (j = 0; j < node_cnt; j++) { + osmids.push_back(random() % (1 << 31)); + } + gettimeofday(&start, NULL); + cache->get_list(nodes,osmids); + gettimeofday(&stop, NULL); + double duration = ((stop.tv_sec - start.tv_sec)*1000000.0 + (stop.tv_usec - start.tv_usec))/1000000.0; + printf("Process %i: Got nodes in %f at a rate of %f/s\n", process_number, duration, node_cnt / duration); + nodes.clear(); + osmids.clear(); + } + gettimeofday(&stop_overall, NULL); + double duration = ((stop_overall.tv_sec - start_overall.tv_sec)*1000000.0 + (stop_overall.tv_usec - start_overall.tv_usec))/1000000.0; + printf("Process %i: Got a total of nodes in %f at a rate of %f/s\n", process_number, duration, node_cnt_total / duration); +} + +int main(int argc, char *argv[]) { + int i,p; + options_t options; + osmNode node; + nodelist_t nodes; + struct timeval start; + idlist_t osmids; + int node_cnt; + options.append = 1; + options.flat_node_cache_enabled = 1; + options.flat_node_file = argv[1]; + boost::shared_ptr ram_cache(new node_ram_cache(0, 10, options.scale)); + boost::shared_ptr cache; + + + if (argc > 3) { + cache.reset(new node_persistent_cache(&options, 1, true, ram_cache)); + node_cnt = argc - 2; + for (i = 0; i < node_cnt; i++) { + osmids.push_back(strtoosmid(argv[2 + i], NULL, 10)); + } + cache->get_list(nodes, osmids); + for (i = 0; i < node_cnt; i++) { + printf("lat: %f / lon: %f\n", nodes[i].lat, nodes[i].lon); + } + } else if (argc == 2) { + char * state = (char *)malloc(sizeof(char)* 128); + gettimeofday(&start, NULL); + initstate(start.tv_usec, state, 8); + setstate(state); + + printf("Testing mode\n"); + cache.reset(new node_persistent_cache(&options, 1, true, ram_cache)); + test_get_node_list(cache, 10, 200, 0); + cache.reset(); +#ifdef HAVE_FORK + printf("Testing using multiple processes\n"); + int noProcs = 4; + int pid; + for (p = 1; p < noProcs; p++) { + pid=fork(); + if (pid==0) { + break; + } + if (pid==-1) { + fprintf(stderr,"WARNING: Failed to fork helper processes. Falling back to only using %i \n", p); + exit(1); + } + } + gettimeofday(&start, NULL); + initstate(start.tv_usec, state, 8); + setstate(state); + cache.reset(new node_persistent_cache(&options, 1, true, ram_cache)); + test_get_node_list(cache, 10,200,p); + + if (pid == 0) { + cache.reset(); + fprintf(stderr,"Exiting process %i\n", p); + exit(0); + } else { + for (p = 0; p < noProcs; p++) wait(NULL); + } + free(state); + fprintf(stderr, "\nAll child processes exited\n"); +#endif + } else { + cache.reset(new node_persistent_cache(&options, 1, true, ram_cache)); + if (strstr(argv[2],",") == NULL) { + cache->get(&node, strtoosmid(argv[2], NULL, 10)); + printf("lat: %f / lon: %f\n", node.lat, node.lon); + } else { + char * node_list = (char *)malloc(sizeof(char) * (strlen(argv[2]) + 1)); + strcpy(node_list,argv[2]); + node_cnt = 1; + strtok(node_list,","); + while (strtok(NULL,",") != NULL) node_cnt++; + printf("Processing %i nodes\n", node_cnt); + strcpy(node_list,argv[2]); + osmids.push_back(strtoosmid(strtok(node_list,","), NULL, 10)); + for (i = 1; i < node_cnt; i++) { + char * tmp = strtok(NULL,","); + osmids.push_back(strtoosmid(tmp, NULL, 10)); + } + cache->get_list(nodes,osmids); + for (i = 0; i < node_cnt; i++) { + printf("lat: %f / lon: %f\n", nodes[i].lat, nodes[i].lon); + } + } + } + + + cache.reset(); + ram_cache.reset(); + + return 0; +} diff -Nru osm2pgsql-0.82.0/node-ram-cache.c osm2pgsql-0.88.1/node-ram-cache.c --- osm2pgsql-0.82.0/node-ram-cache.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/node-ram-cache.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,473 +0,0 @@ -/* Implements a node cache in ram, for the middle layers to use. - * It uses two different storage methods, one optimized for dense - * nodes (with respect to id) and the other for sparse representations. -*/ - -#include "config.h" - -#include -#include -#include -#include -#include - -#include "osmtypes.h" -#include "middle.h" -#include "node-ram-cache.h" - - - - - -/* Here we use a similar storage structure as middle-ram, except we allow - * the array to be lossy so we can cap the total memory usage. Hence it is a - * combination of a sparse array with a priority queue - * - * Like middle-ram we have a number of blocks all storing PER_BLOCK - * ramNodes. However, here we also track the number of nodes in each block. - * Seperately we have a priority queue like structure when maintains a list - * of all the used block so we can easily find the block with the least - * nodes. The cache has two phases: - * - * Phase 1: Loading initially, usedBlocks < maxBlocks. In this case when a - * new block is needed we simply allocate it and put it in - * queue[usedBlocks-1] which is the bottom of the tree. Every node added - * increases it's usage. When we move onto the next block we percolate this - * block up the queue until it reaches its correct position. The invariant - * is that the priority tree is complete except for this last node. We do - * not permit adding nodes to any other block to preserve this invariant. - * - * Phase 2: Once we've reached the maximum number of blocks permitted, we - * change so that the block currently be inserted into is at the top of the - * tree. When a new block is needed we take the one at the end of the queue, - * as it is the one with the least number of nodes in it. When we move onto - * the next block we first push the just completed block down to it's - * correct position in the queue and then reuse the block that now at the - * head. - * - * The result being that at any moment we have in memory the top maxBlock - * blocks in terms of number of nodes in memory. This should maximize the - * number of hits in lookups. - * - * Complexity: - * Insert node: O(1) - * Lookup node: O(1) - * Add new block: O(log usedBlocks) - * Reuse old block: O(log maxBlocks) - */ - - - -static int allocStrategy = ALLOC_DENSE; - -#define BLOCK_SHIFT 10 -#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT) -#define NUM_BLOCKS (((osmid_t)1) << (36 - BLOCK_SHIFT)) - -#define SAFETY_MARGIN 1024*PER_BLOCK*sizeof(struct ramNode) - -static struct ramNodeBlock *blocks; -static int usedBlocks; -/* Note: maxBlocks *must* be odd, to make sure the priority queue has no nodes with only one child */ -static int maxBlocks = 0; -static void *blockCache = NULL; - -static struct ramNodeBlock **queue; - - -static struct ramNodeID *sparseBlock; -static int64_t maxSparseTuples = 0; -static int64_t sizeSparseTuples = 0; - - -static int64_t cacheUsed, cacheSize; -static osmid_t storedNodes, totalNodes; -int nodesCacheHits, nodesCacheLookups; - -static int warn_node_order; - -static int ram_cache_nodes_get_sparse(struct osmNode *out, osmid_t id); - -static int id2block(osmid_t id) -{ - /* + NUM_BLOCKS/2 allows for negative IDs */ - return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2; -} - -static int id2offset(osmid_t id) -{ - return id & (PER_BLOCK-1); -} - -static osmid_t block2id(int block, int offset) -{ - return (((osmid_t) block - NUM_BLOCKS/2) << BLOCK_SHIFT) + (osmid_t) offset; -} - -#define Swap(a,b) { struct ramNodeBlock * __tmp = a; a = b; b = __tmp; } - -static void percolate_up( int pos ) -{ - int i = pos; - while( i > 0 ) - { - int parent = (i-1)>>1; - if( queue[i]->used < queue[parent]->used ) - { - Swap( queue[i], queue[parent] ) - i = parent; - } - else - break; - } -} - -static void *next_chunk(size_t count, size_t size) { - if ( (allocStrategy & ALLOC_DENSE_CHUNK) == 0 ) { - static size_t pos = 0; - void *result; - pos += count * size; - result = blockCache + cacheSize - pos + SAFETY_MARGIN; - - return result; - } else { - return calloc(PER_BLOCK, sizeof(struct ramNode)); - } -} - - -static int ram_cache_nodes_set_sparse(osmid_t id, double lat, double lon, struct keyval *tags UNUSED) { - if ((sizeSparseTuples > maxSparseTuples) || ( cacheUsed > cacheSize)) { - if ((allocStrategy & ALLOC_LOSSY) > 0) - return 1; - else { - fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n"); - exit_nicely(); - } - } - sparseBlock[sizeSparseTuples].id = id; -#ifdef FIXED_POINT - sparseBlock[sizeSparseTuples].coord.lat = DOUBLE_TO_FIX(lat); - sparseBlock[sizeSparseTuples].coord.lon = DOUBLE_TO_FIX(lon); -#else - sparseBlock[sizeSparseTuples].coord.lat = lat; - sparseBlock[sizeSparseTuples].coord.lon = lon; -#endif - sizeSparseTuples++; - cacheUsed += sizeof(struct ramNodeID); - storedNodes++; - return 0; -} - -static int ram_cache_nodes_set_dense(osmid_t id, double lat, double lon, struct keyval *tags UNUSED) { - int block = id2block(id); - int offset = id2offset(id); - int i = 0; - - if (!blocks[block].nodes) { - if (((allocStrategy & ALLOC_SPARSE) > 0) && ( usedBlocks < maxBlocks) && ( cacheUsed > cacheSize)) { - /* TODO: It is more memory efficient to drop nodes from the sparse node cache than from the dense node cache */ - } - if ((usedBlocks < maxBlocks ) && (cacheUsed < cacheSize)) { - /* if usedBlocks > 0 then the previous block is used up. Need to correctly handle it. */ - if ( usedBlocks > 0 ) { - /* If sparse allocation is also set, then check if the previous block has sufficient density - * to store it in dense representation. If not, push all elements of the block - * to the sparse node cache and reuse memory of the previous block for the current block */ - if ( ((allocStrategy & ALLOC_SPARSE) == 0) || - ((queue[usedBlocks - 1]->used / (double)(1<< BLOCK_SHIFT)) > - (sizeof(struct ramNode) / (double)sizeof(struct ramNodeID)))) { - /* Block has reached the level to keep it in dense representation */ - /* We've just finished with the previous block, so we need to percolate it up the queue to its correct position */ - /* Upto log(usedBlocks) iterations */ - percolate_up( usedBlocks-1 ); - blocks[block].nodes = next_chunk(PER_BLOCK, sizeof(struct ramNode)); - } else { - /* previous block was not dense enough, so push it into the sparse node cache instead */ - for (i = 0; i < (1 << BLOCK_SHIFT); i++) { - if (queue[usedBlocks -1]->nodes[i].lat || queue[usedBlocks -1]->nodes[i].lon) { - ram_cache_nodes_set_sparse(block2id(queue[usedBlocks - 1]->block_offset,i), -#ifdef FIXED_POINT - FIX_TO_DOUBLE(queue[usedBlocks -1]->nodes[i].lat), - FIX_TO_DOUBLE(queue[usedBlocks -1]->nodes[i].lon), -#else - queue[usedBlocks -1]->nodes[i].lat, - queue[usedBlocks -1]->nodes[i].lon, -#endif - NULL); - } - } - /* reuse previous block, as it's content is now in the dense representation */ - storedNodes -= queue[usedBlocks - 1]->used; - blocks[block].nodes = queue[usedBlocks - 1]->nodes; - blocks[queue[usedBlocks - 1]->block_offset].nodes = NULL; - memset( blocks[block].nodes, 0, PER_BLOCK * sizeof(struct ramNode) ); - usedBlocks--; - cacheUsed -= PER_BLOCK * sizeof(struct ramNode); - } - } else { - blocks[block].nodes = next_chunk(PER_BLOCK, sizeof(struct ramNode)); - } - - blocks[block].used = 0; - blocks[block].block_offset = block; - if (!blocks[block].nodes) { - fprintf(stderr, "Error allocating nodes\n"); - exit_nicely(); - } - queue[usedBlocks] = &blocks[block]; - usedBlocks++; - cacheUsed += PER_BLOCK * sizeof(struct ramNode); - - - /* If we've just used up the last possible block we enter the - * transition and we change the invariant. To do this we percolate - * the newly allocated block straight to the head */ - if (( usedBlocks == maxBlocks ) || ( cacheUsed > cacheSize )) - percolate_up( usedBlocks-1 ); - } else { - if ((allocStrategy & ALLOC_LOSSY) == 0) { - fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n"); - exit_nicely(); - } - /* We've reached the maximum number of blocks, so now we push the - * current head of the tree down to the right level to restore the - * priority queue invariant. Upto log(maxBlocks) iterations */ - - i=0; - while( 2*i+1 < usedBlocks - 1 ) { - if( queue[2*i+1]->used <= queue[2*i+2]->used ) { - if( queue[i]->used > queue[2*i+1]->used ) { - Swap( queue[i], queue[2*i+1] ); - i = 2*i+1; - } - else - break; - } else { - if( queue[i]->used > queue[2*i+2]->used ) { - Swap( queue[i], queue[2*i+2] ); - i = 2*i+2; - } else - break; - } - } - /* Now the head of the queue is the smallest, so it becomes our replacement candidate */ - blocks[block].nodes = queue[0]->nodes; - blocks[block].used = 0; - memset( blocks[block].nodes, 0, PER_BLOCK * sizeof(struct ramNode) ); - - /* Clear old head block and point to new block */ - storedNodes -= queue[0]->used; - queue[0]->nodes = NULL; - queue[0]->used = 0; - queue[0] = &blocks[block]; - } - } else { - /* Insert into an existing block. We can't allow this in general or it - * will break the invariant. However, it will work fine if all the - * nodes come in numerical order, which is the common case */ - - int expectedpos; - if (( usedBlocks < maxBlocks ) && (cacheUsed < cacheSize)) - expectedpos = usedBlocks-1; - else - expectedpos = 0; - - if( queue[expectedpos] != &blocks[block] ) { - if (!warn_node_order) { - fprintf( stderr, "WARNING: Found Out of order node %" PRIdOSMID " (%d,%d) - this will impact the cache efficiency\n", id, block, offset ); - warn_node_order++; - } - return 1; - } - } - -#ifdef FIXED_POINT - blocks[block].nodes[offset].lat = DOUBLE_TO_FIX(lat); - blocks[block].nodes[offset].lon = DOUBLE_TO_FIX(lon); -#else - blocks[block].nodes[offset].lat = lat; - blocks[block].nodes[offset].lon = lon; -#endif - blocks[block].used++; - storedNodes++; - return 0; -} - - -static int ram_cache_nodes_get_sparse(struct osmNode *out, osmid_t id) { - int64_t pivotPos = sizeSparseTuples >> 1; - int64_t minPos = 0; - int64_t maxPos = sizeSparseTuples; - - while (minPos <= maxPos) { - if ( sparseBlock[pivotPos].id == id ) { -#ifdef FIXED_POINT - out->lat = FIX_TO_DOUBLE(sparseBlock[pivotPos].coord.lat); - out->lon = FIX_TO_DOUBLE(sparseBlock[pivotPos].coord.lon); -#else - out->lat = sparseBlock[pivotPos].coord.lat; - out->lon = sparseBlock[pivotPos].coord.lon; -#endif - return 0; - } - if ( (pivotPos == minPos) || (pivotPos == maxPos)) return 1; - - if ( sparseBlock[pivotPos].id > id ) { - maxPos = pivotPos; - pivotPos = minPos + ((maxPos - minPos) >> 1); - } else { - minPos = pivotPos; - pivotPos = minPos + ((maxPos - minPos) >> 1); - } - } - - return 1; -} - -static int ram_cache_nodes_get_dense(struct osmNode *out, osmid_t id) { - int block = id2block(id); - int offset = id2offset(id); - - if (!blocks[block].nodes) - return 1; - - if (!blocks[block].nodes[offset].lat && !blocks[block].nodes[offset].lon) - return 1; - -#ifdef FIXED_POINT - out->lat = FIX_TO_DOUBLE(blocks[block].nodes[offset].lat); - out->lon = FIX_TO_DOUBLE(blocks[block].nodes[offset].lon); -#else - out->lat = blocks[block].nodes[offset].lat; - out->lon = blocks[block].nodes[offset].lon; -#endif - - return 0; -} - - -void init_node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale ) { - - blockCache = 0; - cacheUsed = 0; - cacheSize = (int64_t)cacheSizeMB*(1024*1024); - /* How much we can fit, and make sure it's odd */ - maxBlocks = (cacheSize/(PER_BLOCK*sizeof(struct ramNode))) | 1; - maxSparseTuples = (cacheSize/sizeof(struct ramNodeID)) | 1; - - allocStrategy = strategy; - scale = fixpointscale; - - if ((allocStrategy & ALLOC_DENSE) > 0 ) { - fprintf(stderr, "Allocating memory for dense node cache\n"); - blocks = calloc(NUM_BLOCKS,sizeof(struct ramNodeBlock)); - if (!blocks) { - fprintf(stderr, "Out of memory for node cache dense index, try using \"--cache-strategy sparse\" instead \n"); - exit_nicely(); - } - queue = calloc( maxBlocks,sizeof(struct ramNodeBlock *) ); - /* Use this method of allocation if virtual memory is limited, - * or if OS allocs physical memory right away, rather than page by page - * once it is needed. - */ - if( (allocStrategy & ALLOC_DENSE_CHUNK) > 0 ) { - fprintf(stderr, "Allocating dense node cache in block sized chunks\n"); - if (!queue) { - fprintf(stderr, "Out of memory, reduce --cache size\n"); - exit_nicely(); - } - } else { - fprintf(stderr, "Allocating dense node cache in one big chunk\n"); - blockCache = calloc(maxBlocks + 1024,PER_BLOCK * sizeof(struct ramNode)); - if (!queue || !blockCache) { - fprintf(stderr, "Out of memory for dense node cache, reduce --cache size\n"); - exit_nicely(); - } - } - } - - /* Allocate the full amount of memory given by --cache parameter in one go. - * If both dense and sparse cache alloc is set, this will allocate up to twice - * as much virtual memory as specified by --cache. This relies on the OS doing - * lazy allocation of physical RAM. Extra accounting during setting of nodes is done - * to ensure physical RAM usage should roughly be no more than --cache - */ - - if ((allocStrategy & ALLOC_SPARSE) > 0 ) { - fprintf(stderr, "Allocating memory for sparse node cache\n"); - if (!blockCache) { - sparseBlock = calloc(maxSparseTuples,sizeof(struct ramNodeID)); - } else { - fprintf(stderr, "Sharing dense sparse\n"); - sparseBlock = blockCache; - } - if (!sparseBlock) { - fprintf(stderr, "Out of memory for sparse node cache, reduce --cache size\n"); - exit_nicely(); - } - } - -#ifdef __MINGW_H - fprintf( stderr, "Node-cache: cache=%ldMB, maxblocks=%d*%d, allocation method=%i\n", (cacheSize >> 20), maxBlocks, PER_BLOCK*sizeof(struct ramNode), allocStrategy ); -#else - fprintf( stderr, "Node-cache: cache=%ldMB, maxblocks=%d*%zd, allocation method=%i\n", (cacheSize >> 20), maxBlocks, PER_BLOCK*sizeof(struct ramNode), allocStrategy ); -#endif -} - -void free_node_ram_cache() { - int i; - fprintf( stderr, "node cache: stored: %" PRIdOSMID "(%.2f%%), storage efficiency: %.2f%% (dense blocks: %i, sparse nodes: %li), hit rate: %.2f%%\n", - storedNodes, 100.0f*storedNodes/totalNodes, 100.0f*storedNodes*sizeof(struct ramNode)/cacheUsed, - usedBlocks, sizeSparseTuples, - 100.0f*nodesCacheHits/nodesCacheLookups ); - - if ( (allocStrategy & ALLOC_DENSE) > 0 ) { - if ( (allocStrategy & ALLOC_DENSE_CHUNK) > 0 ) { - for( i=0; inodes); - queue[i]->nodes = NULL; - } - } else { - free(blockCache); - blockCache = 0; - } - free(queue); - } - if ( ((allocStrategy & ALLOC_SPARSE) > 0) && ((allocStrategy & ALLOC_DENSE) == 0)) { - free(sparseBlock); - } -} - -int ram_cache_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags UNUSED) { - totalNodes++; - /* if ALLOC_DENSE and ALLOC_SPARSE are set, send it through - * ram_nodes_set_dense. If a block is non dense, it will automatically - * get pushed to the sparse cache if a block is sparse and ALLOC_SPARSE is set - */ - if ( (allocStrategy & ALLOC_DENSE) > 0 ) { - return ram_cache_nodes_set_dense(id, lat, lon, tags); - } - if ( (allocStrategy & ALLOC_SPARSE) > 0 ) - return ram_cache_nodes_set_sparse(id, lat, lon, tags); - return 1; -} - -int ram_cache_nodes_get(struct osmNode *out, osmid_t id) { - nodesCacheLookups++; - - if ((allocStrategy & ALLOC_DENSE) > 0) { - if (ram_cache_nodes_get_dense(out,id) == 0) { - nodesCacheHits++; - return 0; - } - } - if ((allocStrategy & ALLOC_SPARSE) > 0) { - if (ram_cache_nodes_get_sparse(out,id) == 0) { - nodesCacheHits++; - return 0; - } - } - - return 1; -} diff -Nru osm2pgsql-0.82.0/node-ram-cache.cpp osm2pgsql-0.88.1/node-ram-cache.cpp --- osm2pgsql-0.82.0/node-ram-cache.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/node-ram-cache.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,432 @@ +/* Implements a node cache in ram, for the middle layers to use. + * It uses two different storage methods, one optimized for dense + * nodes (with respect to id) and the other for sparse representations. +*/ + +#include "config.h" + +#include +#include +#include + +#include "osmtypes.hpp" +#include "node-ram-cache.hpp" +#include "util.hpp" + +/* Here we use a similar storage structure as middle-ram, except we allow + * the array to be lossy so we can cap the total memory usage. Hence it is a + * combination of a sparse array with a priority queue + * + * Like middle-ram we have a number of blocks all storing PER_BLOCK + * ramNodes. However, here we also track the number of nodes in each block. + * Seperately we have a priority queue like structure when maintains a list + * of all the used block so we can easily find the block with the least + * nodes. The cache has two phases: + * + * Phase 1: Loading initially, usedBlocks < maxBlocks. In this case when a + * new block is needed we simply allocate it and put it in + * queue[usedBlocks-1] which is the bottom of the tree. Every node added + * increases it's usage. When we move onto the next block we percolate this + * block up the queue until it reaches its correct position. The invariant + * is that the priority tree is complete except for this last node. We do + * not permit adding nodes to any other block to preserve this invariant. + * + * Phase 2: Once we've reached the maximum number of blocks permitted, we + * change so that the block currently be inserted into is at the top of the + * tree. When a new block is needed we take the one at the end of the queue, + * as it is the one with the least number of nodes in it. When we move onto + * the next block we first push the just completed block down to it's + * correct position in the queue and then reuse the block that now at the + * head. + * + * The result being that at any moment we have in memory the top maxBlock + * blocks in terms of number of nodes in memory. This should maximize the + * number of hits in lookups. + * + * Complexity: + * Insert node: O(1) + * Lookup node: O(1) + * Add new block: O(log usedBlocks) + * Reuse old block: O(log maxBlocks) + */ + + + +#define BLOCK_SHIFT 10 +#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT) +#define NUM_BLOCKS (((osmid_t)1) << (36 - BLOCK_SHIFT)) + +#define SAFETY_MARGIN 1024*PER_BLOCK*sizeof(ramNode) + +int ramNode::scale; + +static int32_t id2block(osmid_t id) +{ + /* + NUM_BLOCKS/2 allows for negative IDs */ + return (id >> BLOCK_SHIFT) + NUM_BLOCKS/2; +} + +static int id2offset(osmid_t id) +{ + return id & (PER_BLOCK-1); +} + +static osmid_t block2id(int32_t block, int offset) +{ + return (((osmid_t) block - NUM_BLOCKS/2) << BLOCK_SHIFT) + (osmid_t) offset; +} + +#define Swap(a,b) { ramNodeBlock * __tmp = a; a = b; b = __tmp; } + +void node_ram_cache::percolate_up( int pos ) +{ + int i = pos; + while( i > 0 ) + { + int parent = (i-1)>>1; + if( queue[i]->used() < queue[parent]->used() ) + { + Swap( queue[i], queue[parent] ) + i = parent; + } + else + break; + } +} + +ramNode *node_ram_cache::next_chunk() { + if ( (allocStrategy & ALLOC_DENSE_CHUNK) == 0 ) { + // allocate starting from the upper end of the block cache + blockCachePos += PER_BLOCK * sizeof(ramNode); + char *result = blockCache + cacheSize - blockCachePos + SAFETY_MARGIN; + + return new(result) ramNode[PER_BLOCK]; + } else { + return new ramNode[PER_BLOCK]; + } +} + + +int node_ram_cache::set_sparse(osmid_t id, const ramNode &coord) { + // Sparse cache depends on ordered nodes, reject out-of-order ids. + // Also check that there is still space. + if ((maxSparseId && id < maxSparseId) + || (sizeSparseTuples > maxSparseTuples) + || ( cacheUsed > cacheSize)) { + if ((allocStrategy & ALLOC_LOSSY) > 0) + return 1; + else { + fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n"); + util::exit_nicely(); + } + } + maxSparseId = id; + sparseBlock[sizeSparseTuples].id = id; + sparseBlock[sizeSparseTuples].coord = coord; + + sizeSparseTuples++; + cacheUsed += sizeof(ramNodeID); + storedNodes++; + return 0; +} + +int node_ram_cache::set_dense(osmid_t id, const ramNode &coord) { + int32_t const block = id2block(id); + int const offset = id2offset(id); + + if (maxBlocks == 0) return 1; + + if (!blocks[block].nodes) { + if (((allocStrategy & ALLOC_SPARSE) > 0) && ( usedBlocks < maxBlocks) && ( cacheUsed > cacheSize)) { + /* TODO: It is more memory efficient to drop nodes from the sparse node cache than from the dense node cache */ + } + if ((usedBlocks < maxBlocks ) && (cacheUsed < cacheSize)) { + /* if usedBlocks > 0 then the previous block is used up. Need to correctly handle it. */ + if ( usedBlocks > 0 ) { + /* If sparse allocation is also set, then check if the previous block has sufficient density + * to store it in dense representation. If not, push all elements of the block + * to the sparse node cache and reuse memory of the previous block for the current block */ + if ( ((allocStrategy & ALLOC_SPARSE) == 0) || + ((queue[usedBlocks - 1]->used() / (double)(1<< BLOCK_SHIFT)) > + (sizeof(ramNode) / (double)sizeof(ramNodeID)))) { + /* Block has reached the level to keep it in dense representation */ + /* We've just finished with the previous block, so we need to percolate it up the queue to its correct position */ + /* Upto log(usedBlocks) iterations */ + percolate_up( usedBlocks-1 ); + blocks[block].nodes = next_chunk(); + } else { + /* previous block was not dense enough, so push it into the sparse node cache instead */ + for (int i = 0; i < (1 << BLOCK_SHIFT); i++) { + if (queue[usedBlocks -1]->nodes[i].is_valid()) { + set_sparse(block2id(queue[usedBlocks - 1]->block_offset, i), + queue[usedBlocks -1]->nodes[i]); + queue[usedBlocks -1]->nodes[i] = ramNode(); // invalidate + } + } + /* reuse previous block, as its content is now in the sparse representation */ + storedNodes -= queue[usedBlocks - 1]->used(); + blocks[block].nodes = queue[usedBlocks - 1]->nodes; + blocks[queue[usedBlocks - 1]->block_offset].nodes = NULL; + usedBlocks--; + cacheUsed -= PER_BLOCK * sizeof(ramNode); + } + } else { + blocks[block].nodes = next_chunk(); + } + + blocks[block].reset_used(); + blocks[block].block_offset = block; + if (!blocks[block].nodes) { + fprintf(stderr, "Error allocating nodes\n"); + util::exit_nicely(); + } + queue[usedBlocks] = &blocks[block]; + usedBlocks++; + cacheUsed += PER_BLOCK * sizeof(ramNode); + + /* If we've just used up the last possible block we enter the + * transition and we change the invariant. To do this we percolate + * the newly allocated block straight to the head */ + if (( usedBlocks == maxBlocks ) || ( cacheUsed > cacheSize )) + percolate_up( usedBlocks-1 ); + } else { + if ((allocStrategy & ALLOC_LOSSY) == 0) { + fprintf(stderr, "\nNode cache size is too small to fit all nodes. Please increase cache size\n"); + util::exit_nicely(); + } + /* We've reached the maximum number of blocks, so now we push the + * current head of the tree down to the right level to restore the + * priority queue invariant. Upto log(maxBlocks) iterations */ + + int i = 0; + while( 2*i+1 < usedBlocks - 1 ) { + if( queue[2*i+1]->used() <= queue[2*i+2]->used() ) { + if( queue[i]->used() > queue[2*i+1]->used() ) { + Swap( queue[i], queue[2*i+1] ); + i = 2*i+1; + } + else + break; + } else { + if( queue[i]->used() > queue[2*i+2]->used() ) { + Swap( queue[i], queue[2*i+2] ); + i = 2*i+2; + } else + break; + } + } + /* Now the head of the queue is the smallest, so it becomes our replacement candidate */ + blocks[block].nodes = queue[0]->nodes; + blocks[block].reset_used(); + new(blocks[block].nodes) ramNode[PER_BLOCK]; + + /* Clear old head block and point to new block */ + storedNodes -= queue[0]->used(); + queue[0]->nodes = NULL; + queue[0]->reset_used(); + queue[0] = &blocks[block]; + } + } else { + /* Insert into an existing block. We can't allow this in general or it + * will break the invariant. However, it will work fine if all the + * nodes come in numerical order, which is the common case */ + + int expectedpos; + if (( usedBlocks < maxBlocks ) && (cacheUsed < cacheSize)) + expectedpos = usedBlocks-1; + else + expectedpos = 0; + + if( queue[expectedpos] != &blocks[block] ) { + if (!warn_node_order) { + fprintf( stderr, "WARNING: Found Out of order node %" PRIdOSMID " (%d,%d) - this will impact the cache efficiency\n", id, block, offset ); + warn_node_order++; + } + return 1; + } + } + + blocks[block].nodes[offset] = coord; + blocks[block].inc_used(); + storedNodes++; + return 0; +} + + +int node_ram_cache::get_sparse(osmNode *out, osmid_t id) { + int64_t pivotPos = sizeSparseTuples >> 1; + int64_t minPos = 0; + int64_t maxPos = sizeSparseTuples; + + while (minPos <= maxPos) { + if ( sparseBlock[pivotPos].id == id ) { + out->lat = sparseBlock[pivotPos].coord.lat(); + out->lon = sparseBlock[pivotPos].coord.lon(); + return 0; + } + if ( (pivotPos == minPos) || (pivotPos == maxPos)) return 1; + + if ( sparseBlock[pivotPos].id > id ) { + maxPos = pivotPos; + pivotPos = minPos + ((maxPos - minPos) >> 1); + } else { + minPos = pivotPos; + pivotPos = minPos + ((maxPos - minPos) >> 1); + } + } + + return 1; +} + +int node_ram_cache::get_dense(osmNode *out, osmid_t id) { + int32_t const block = id2block(id); + int const offset = id2offset(id); + + if (!blocks[block].nodes) + return 1; + + if (!blocks[block].nodes[offset].is_valid()) + return 1; + + out->lat = blocks[block].nodes[offset].lat(); + out->lon = blocks[block].nodes[offset].lon(); + + return 0; +} + + +node_ram_cache::node_ram_cache( int strategy, int cacheSizeMB, int fixpointscale ) + : allocStrategy(ALLOC_DENSE), blocks(NULL), usedBlocks(0), + maxBlocks(0), blockCache(NULL), queue(NULL), sparseBlock(NULL), + maxSparseTuples(0), sizeSparseTuples(0), maxSparseId(0), cacheUsed(0), + cacheSize(0), storedNodes(0), totalNodes(0), nodesCacheHits(0), + nodesCacheLookups(0), warn_node_order(0) { + + ramNode::scale = fixpointscale; + blockCache = 0; + blockCachePos = 0; + cacheUsed = 0; + cacheSize = (int64_t)cacheSizeMB*(1024*1024); + /* How much we can fit, and make sure it's odd */ + maxBlocks = (cacheSize/(PER_BLOCK*sizeof(ramNode))); + maxSparseTuples = (cacheSize/sizeof(ramNodeID))+1; + + allocStrategy = strategy; + + if ((allocStrategy & ALLOC_DENSE) > 0 ) { + fprintf(stderr, "Allocating memory for dense node cache\n"); + blocks = (ramNodeBlock *)calloc(NUM_BLOCKS,sizeof(ramNodeBlock)); + if (!blocks) { + fprintf(stderr, "Out of memory for node cache dense index, try using \"--cache-strategy sparse\" instead \n"); + util::exit_nicely(); + } + queue = (ramNodeBlock **)calloc( maxBlocks,sizeof(ramNodeBlock *) ); + /* Use this method of allocation if virtual memory is limited, + * or if OS allocs physical memory right away, rather than page by page + * once it is needed. + */ + if( (allocStrategy & ALLOC_DENSE_CHUNK) > 0 ) { + fprintf(stderr, "Allocating dense node cache in block sized chunks\n"); + if (!queue) { + fprintf(stderr, "Out of memory, reduce --cache size\n"); + util::exit_nicely(); + } + } else { + fprintf(stderr, "Allocating dense node cache in one big chunk\n"); + blockCache = (char *)malloc((maxBlocks + 1024) * PER_BLOCK * sizeof(ramNode)); + if (!queue || !blockCache) { + fprintf(stderr, "Out of memory for dense node cache, reduce --cache size\n"); + util::exit_nicely(); + } + } + } + + /* Allocate the full amount of memory given by --cache parameter in one go. + * If both dense and sparse cache alloc is set, this will allocate up to twice + * as much virtual memory as specified by --cache. This relies on the OS doing + * lazy allocation of physical RAM. Extra accounting during setting of nodes is done + * to ensure physical RAM usage should roughly be no more than --cache + */ + + if ((allocStrategy & ALLOC_SPARSE) > 0 ) { + fprintf(stderr, "Allocating memory for sparse node cache\n"); + if (!blockCache) { + sparseBlock = (ramNodeID *)malloc(maxSparseTuples * sizeof(ramNodeID)); + } else { + fprintf(stderr, "Sharing dense sparse\n"); + sparseBlock = (ramNodeID *)blockCache; + } + if (!sparseBlock) { + fprintf(stderr, "Out of memory for sparse node cache, reduce --cache size\n"); + util::exit_nicely(); + } + } + +#ifdef __MINGW_H + fprintf( stderr, "Node-cache: cache=%ldMB, maxblocks=%d*%d, allocation method=%i\n", (cacheSize >> 20), maxBlocks, PER_BLOCK*sizeof(ramNode), allocStrategy ); +#else + fprintf( stderr, "Node-cache: cache=%ldMB, maxblocks=%d*%zd, allocation method=%i\n", (cacheSize >> 20), maxBlocks, PER_BLOCK*sizeof(ramNode), allocStrategy ); +#endif +} + +node_ram_cache::~node_ram_cache() { + fprintf( stderr, "node cache: stored: %" PRIdOSMID "(%.2f%%), storage efficiency: %.2f%% (dense blocks: %i, sparse nodes: %li), hit rate: %.2f%%\n", + storedNodes, 100.0f*storedNodes/totalNodes, 100.0f*storedNodes*sizeof(ramNode)/cacheUsed, + usedBlocks, sizeSparseTuples, + 100.0f*nodesCacheHits/nodesCacheLookups ); + + if ( (allocStrategy & ALLOC_DENSE) > 0 ) { + if ( (allocStrategy & ALLOC_DENSE_CHUNK) > 0 ) { + for(int i = 0; i < usedBlocks; ++i) { + delete[] queue[i]->nodes; + queue[i]->nodes = NULL; + } + } else { + free(blockCache); + blockCache = 0; + } + free(blocks); + free(queue); + } + if ( ((allocStrategy & ALLOC_SPARSE) > 0) && ((allocStrategy & ALLOC_DENSE) == 0)) { + free(sparseBlock); + } +} + +int node_ram_cache::set(osmid_t id, double lat, double lon, const taglist_t &) { + if ((id > 0 && id >> BLOCK_SHIFT >> 32) || (id < 0 && ~id >> BLOCK_SHIFT >> 32 )) { + fprintf(stderr, "\nAbsolute node IDs must not be larger than %lld (got %lld)\n", + 1ULL << 42, (long long) id); + util::exit_nicely(); + } + totalNodes++; + /* if ALLOC_DENSE and ALLOC_SPARSE are set, send it through + * ram_nodes_set_dense. If a block is non dense, it will automatically + * get pushed to the sparse cache if a block is sparse and ALLOC_SPARSE is set + */ + if ( (allocStrategy & ALLOC_DENSE) > 0 ) { + return set_dense(id, ramNode(lon, lat)); + } + if ( (allocStrategy & ALLOC_SPARSE) > 0 ) + return set_sparse(id, ramNode(lon, lat)); + return 1; +} + +int node_ram_cache::get(osmNode *out, osmid_t id) { + nodesCacheLookups++; + + if ((allocStrategy & ALLOC_DENSE) > 0) { + if (get_dense(out, id) == 0) { + nodesCacheHits++; + return 0; + } + } + if ((allocStrategy & ALLOC_SPARSE) > 0) { + if (get_sparse(out, id) == 0) { + nodesCacheHits++; + return 0; + } + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/node-ram-cache.h osm2pgsql-0.88.1/node-ram-cache.h --- osm2pgsql-0.82.0/node-ram-cache.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/node-ram-cache.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,53 +0,0 @@ -/* Implements the node cache in ram. - * - * There are two different storage strategies, either optimised - * for dense storage of node ids, or for sparse storage as well as - * a strategy to combine both in an optimal way. -*/ - -#ifndef NODE_RAM_CACHE_H -#define NODE_RAM_CACHE_H - -#define ALLOC_SPARSE 1 -#define ALLOC_DENSE 2 -#define ALLOC_DENSE_CHUNK 4 -#define ALLOC_LOSSY 8 - -/* Store +-20,000km Mercator co-ordinates as fixed point 32bit number with maximum precision */ -/* Scale is chosen such that 40,000 * SCALE < 2^32 */ -#define FIXED_POINT -static int scale = 100; -#define DOUBLE_TO_FIX(x) ((int)((x) * scale)) -#define FIX_TO_DOUBLE(x) (((double)x) / scale) - -#define UNUSED __attribute__ ((unused)) - -struct ramNode { -#ifdef FIXED_POINT - int lon; - int lat; -#else - double lon; - double lat; -#endif -}; - -struct ramNodeID { - osmid_t id; - struct ramNode coord; -}; - -struct ramNodeBlock { - struct ramNode *nodes; - osmid_t block_offset; - int used; - int dirty; -}; - - -void init_node_ram_cache(int strategy, int cacheSizeMB, int fixpointscale); -void free_node_ram_cache(); -int ram_cache_nodes_set(osmid_t id, double lat, double lon, struct keyval *tags UNUSED); -int ram_cache_nodes_get(struct osmNode *out, osmid_t id); - -#endif diff -Nru osm2pgsql-0.82.0/node-ram-cache.hpp osm2pgsql-0.88.1/node-ram-cache.hpp --- osm2pgsql-0.82.0/node-ram-cache.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/node-ram-cache.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,144 @@ +/* Implements the node cache in ram. + * + * There are two different storage strategies, either optimised + * for dense storage of node ids, or for sparse storage as well as + * a strategy to combine both in an optimal way. +*/ + +#ifndef NODE_RAM_CACHE_H +#define NODE_RAM_CACHE_H + +#include "config.h" +#include "osmtypes.hpp" + +#include +#include +#include +#include + +#define ALLOC_SPARSE 1 +#define ALLOC_DENSE 2 +#define ALLOC_DENSE_CHUNK 4 +#define ALLOC_LOSSY 8 + +/** + * A set of coordinates, for caching in RAM or on disk. + * + * If FIXED_POINT is enabled, it uses internally a more efficient + * representation as integer. + */ +class ramNode { +public: +#ifdef FIXED_POINT + static int scale; + + /// Default constructor creates an invalid node + ramNode() : _lon(INT_MIN), _lat(INT_MIN) {} + /** + * Standard constructor takes geographic coordinates and saves them + * in the internal node representation. + */ + ramNode(double lon, double lat) : _lon(dbl2fix(lon)), _lat(dbl2fix(lat)) {} + /** + * Internal constructor which takes already encoded nodes. + * + * Used by middle-pgsql which stores encoded nodes in the DB. + */ + ramNode(int lon, int lat) : _lon(lon), _lat(lat) {} + + /// Return true if the node currently stores valid coordinates. + bool is_valid() const { return _lon != INT_MIN; } + /// Return longitude (converting from internal representation) + double lon() const { return fix2dbl(_lon); } + /// Return latitude (converting from internal representation) + double lat() const { return fix2dbl(_lat); } + /// Return internal representation of longitude (for external storage). + int int_lon() const { return _lon; } + /// Return internal representation of latitude (for external storage). + int int_lat() const { return _lat; } + +private: + int _lon; + int _lat; + + int dbl2fix(const double x) const { return x * scale + 0.4; } + double fix2dbl(const int x) const { return (double)x / scale; } +#else +public: + ramNode() : _lat(NAN), _lon(NAN) {} + ramNode(double _lon, double _lat) : _lon(lon), _lat(lat) {} + + bool is_valid() const ( return !std::isnan(_lon); } + double lon() const { return _lon; } + double lat() const { return _lat; } +private: + double _lon; + double _lat; + +#endif +}; + +struct ramNodeID { + osmid_t id; + ramNode coord; +}; + +class ramNodeBlock { +public: + ramNodeBlock() : nodes(NULL), block_offset(-1), _used(0) {} + + void set_dirty() { _used |= 1; } + bool dirty() const { return _used & 1; } + + void reset_used() { _used = 0; } + void inc_used() { _used += 2; } + void dec_used() { _used -= 2; } + void set_used(int used) { _used = (used << 1) || (_used & 1); } + int used() const { return _used >> 1; } + + ramNode *nodes; + int32_t block_offset; +private: + int32_t _used; // 0-bit indicates dirty +}; + +struct node_ram_cache : public boost::noncopyable +{ + node_ram_cache(int strategy, int cacheSizeMB, int fixpointscale); + ~node_ram_cache(); + + int set(osmid_t id, double lat, double lon, const taglist_t &tags); + int get(osmNode *out, osmid_t id); + +private: + void percolate_up( int pos ); + ramNode *next_chunk(); + int set_sparse(osmid_t id, const ramNode &coord); + int set_dense(osmid_t id, const ramNode& coord); + int get_sparse(osmNode *out, osmid_t id); + int get_dense(osmNode *out, osmid_t id); + + int allocStrategy; + + ramNodeBlock *blocks; + int usedBlocks; + /* Note: maxBlocks *must* be odd, to make sure the priority queue has no nodes with only one child */ + int maxBlocks; + char *blockCache; + size_t blockCachePos; + + ramNodeBlock **queue; + + ramNodeID *sparseBlock; + int64_t maxSparseTuples; + int64_t sizeSparseTuples; + osmid_t maxSparseId; + + int64_t cacheUsed, cacheSize; + osmid_t storedNodes, totalNodes; + int nodesCacheHits, nodesCacheLookups; + + int warn_node_order; +}; + +#endif diff -Nru osm2pgsql-0.82.0/options.cpp osm2pgsql-0.88.1/options.cpp --- osm2pgsql-0.82.0/options.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/options.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,548 @@ +#include "options.hpp" +#include "sprompt.hpp" +#include "parse.hpp" + +#include +#ifdef HAVE_LIBGEN_H +#include +#else +#define basename /*SKIP IT*/ +#endif +#include +#include +#include +#include +#include + +namespace +{ + const char * short_options = "ab:cd:KhlmMp:suvU:WH:P:i:IE:C:S:e:o:O:xkjGz:r:V"; + const struct option long_options[] = + { + {"append", 0, 0, 'a'}, + {"bbox", 1, 0, 'b'}, + {"create", 0, 0, 'c'}, + {"database", 1, 0, 'd'}, + {"latlong", 0, 0, 'l'}, + {"verbose", 0, 0, 'v'}, + {"slim", 0, 0, 's'}, + {"prefix", 1, 0, 'p'}, + {"proj", 1, 0, 'E'}, + {"merc", 0, 0, 'm'}, + {"utf8-sanitize", 0, 0, 'u'}, + {"cache", 1, 0, 'C'}, + {"username", 1, 0, 'U'}, + {"password", 0, 0, 'W'}, + {"host", 1, 0, 'H'}, + {"port", 1, 0, 'P'}, + {"tablespace-index", 1, 0, 'i'}, + {"tablespace-slim-data", 1, 0, 200}, + {"tablespace-slim-index", 1, 0, 201}, + {"tablespace-main-data", 1, 0, 202}, + {"tablespace-main-index", 1, 0, 203}, + {"help", 0, 0, 'h'}, + {"style", 1, 0, 'S'}, + {"expire-tiles", 1, 0, 'e'}, + {"expire-output", 1, 0, 'o'}, + {"output", 1, 0, 'O'}, + {"extra-attributes", 0, 0, 'x'}, + {"hstore", 0, 0, 'k'}, + {"hstore-all", 0, 0, 'j'}, + {"hstore-column", 1, 0, 'z'}, + {"hstore-match-only", 0, 0, 208}, + {"hstore-add-index",0,0,211}, + {"multi-geometry", 0, 0, 'G'}, + {"keep-coastlines", 0, 0, 'K'}, + {"input-reader", 1, 0, 'r'}, + {"version", 0, 0, 'V'}, + {"disable-parallel-indexing", 0, 0, 'I'}, + {"cache-strategy", 1, 0, 204}, + {"number-processes", 1, 0, 205}, + {"drop", 0, 0, 206}, + {"unlogged", 0, 0, 207}, + {"flat-nodes",1,0,209}, + {"exclude-invalid-polygon",0,0,210}, + {"tag-transform-script",1,0,212}, + {0, 0, 0, 0} + }; + + void short_usage(char *arg0) + { + throw std::runtime_error((boost::format("Usage error. For further information see:\n\t%1% -h|--help\n") % basename(arg0)).str()); + } + + void long_usage(char *arg0, bool verbose = false) + { + const char *name = basename(arg0); + + printf("Usage:\n"); + printf("\t%s [options] planet.osm\n", name); + printf("\t%s [options] planet.osm.{pbf,gz,bz2}\n", name); + printf("\t%s [options] file1.osm file2.osm file3.osm\n", name); + printf("\nThis will import the data from the OSM file(s) into a PostgreSQL database\n"); + printf("suitable for use by the Mapnik renderer.\n\n"); + + printf("%s", "\ + Common options:\n\ + -a|--append Add the OSM file into the database without removing\n\ + existing data.\n\ + -c|--create Remove existing data from the database. This is the\n\ + default if --append is not specified.\n\ + -l|--latlong Store data in degrees of latitude & longitude.\n\ + -m|--merc Store data in proper spherical mercator (default).\n\ + -E|--proj num Use projection EPSG:num.\n\ + -s|--slim Store temporary data in the database. This greatly\n\ + reduces the RAM usage but is much slower. This switch is\n\ + required if you want to update with --append later.\n\ + -S|--style Location of the style file. Defaults to\n"); + printf("\ + %s/default.style.\n", OSM2PGSQL_DATADIR); + printf("%s", "\ + -C|--cache Use up to this many MB for caching nodes (default: 800)\n\ + \n\ + Database options:\n\ + -d|--database The name of the PostgreSQL database to connect\n\ + to (default: gis).\n\ + -U|--username PostgreSQL user name (specify passsword in PGPASS\n\ + environment variable or use -W).\n\ + -W|--password Force password prompt.\n\ + -H|--host Database server host name or socket location.\n\ + -P|--port Database server port.\n"); + + if (verbose) + { + printf("%s", "\ + Hstore options:\n\ + -k|--hstore Add tags without column to an additional hstore\n\ + (key/value) column\n\ + --hstore-match-only Only keep objects that have a value in one of\n\ + the columns (default with --hstore is to keep all objects)\n\ + -j|--hstore-all Add all tags to an additional hstore (key/value) column\n\ + -z|--hstore-column Add an additional hstore (key/value) column containing\n\ + all tags that start with the specified string, eg\n\ + --hstore-column \"name:\" will produce an extra hstore\n\ + column that contains all name:xx tags\n\ + --hstore-add-index Add index to hstore column.\n\ + \n\ + Obsolete options:\n\ + -u|--utf8-sanitize Repair bad UTF8 input data (present in planet\n\ + dumps prior to August 2007). Adds about 10% overhead.\n\ + \n\ + Performance options:\n\ + -i|--tablespace-index The name of the PostgreSQL tablespace where\n\ + all indexes will be created.\n\ + The following options allow more fine-grained control:\n\ + --tablespace-main-data tablespace for main tables\n\ + --tablespace-main-index tablespace for main table indexes\n\ + --tablespace-slim-data tablespace for slim mode tables\n\ + --tablespace-slim-index tablespace for slim mode indexes\n\ + (if unset, use db's default; -i is equivalent to setting\n\ + --tablespace-main-index and --tablespace-slim-index)\n\ + --drop only with --slim: drop temporary tables after import \n\ + (no updates are possible).\n\ + --number-processes Specifies the number of parallel processes \n\ + used for certain operations (default is 1).\n\ + -I|--disable-parallel-indexing Disable indexing all tables concurrently.\n\ + --unlogged Use unlogged tables (lost on crash but faster). \n\ + Requires PostgreSQL 9.1.\n\ + --cache-strategy Specifies the method used to cache nodes in ram.\n\ + Available options are:\n\ + dense: caching strategy optimised for full planet import\n\ + chunk: caching strategy optimised for non-contiguous \n\ + memory allocation\n\ + sparse: caching strategy optimised for small extracts\n\ + optimized: automatically combines dense and sparse \n\ + strategies for optimal storage efficiency. This may\n\ + us twice as much virtual memory, but no more physical \n\ + memory.\n"); + #ifdef __amd64__ + printf("\ + The default is \"optimized\"\n"); + #else + /* use "chunked" as a default in 32 bit compilations, as it is less wasteful of virtual memory than "optimized"*/ + printf("\ + The default is \"sparse\"\n"); + #endif + printf("%s", "\ + --flat-nodes Specifies the flat file to use to persistently store node \n\ + information in slim mode instead of in PostgreSQL.\n\ + This file is a single > 16Gb large file. Only recommended\n\ + for full planet imports. Default is disabled.\n\ + \n\ + Expiry options:\n\ + -e|--expire-tiles [min_zoom-]max_zoom Create a tile expiry list.\n\ + -o|--expire-output filename Output filename for expired tiles list.\n\ + \n\ + Other options:\n\ + -b|--bbox Apply a bounding box filter on the imported data\n\ + Must be specified as: minlon,minlat,maxlon,maxlat\n\ + e.g. --bbox -0.5,51.25,0.5,51.75\n\ + -p|--prefix Prefix for table names (default planet_osm)\n\ + -r|--input-reader Input frontend.\n\ + libxml2 - Parse XML using libxml2. (default)\n"); + #ifdef BUILD_READER_PBF + printf("\ + pbf - OSM binary format.\n"); + #endif + printf("\ + -O|--output Output backend.\n\ + pgsql - Output to a PostGIS database (default)\n\ + multi - Multiple Custom Table Output to a PostGIS \n\ + database (requires style file for configuration)\n\ + gazetteer - Output to a PostGIS database for Nominatim\n\ + null - No output. Useful for testing\n"); + #ifdef HAVE_LUA + printf("\ + --tag-transform-script Specify a lua script to handle tag filtering and normalisation\n\ + The script contains callback functions for nodes, ways and relations, which each\n\ + take a set of tags and returns a transformed, filtered set of tags which are then\n\ + written to the database.\n"); + #endif + printf("\ + -x|--extra-attributes\n\ + Include attributes for each object in the database.\n\ + This includes the username, userid, timestamp and version.\n\ + Requires additional entries in your style file.\n\ + -G|--multi-geometry Generate multi-geometry features in postgresql tables.\n\ + -K|--keep-coastlines Keep coastline data rather than filtering it out.\n\ + By default natural=coastline tagged data will be discarded\n\ + because renderers usually have shape files for them.\n\ + --exclude-invalid-polygon do not import polygons with invalid geometries.\n\ + -h|--help Help information.\n\ + -v|--verbose Verbose output.\n"); + } + else + { + printf("\n"); + printf("A typical command to import a full planet is\n"); + printf(" %s -c -d gis --slim -C -k \\\n", name); + printf(" --flat-nodes planet-latest.osm.pbf\n"); + printf("where\n"); + printf(" is 20000 on machines with 24GB or more RAM \n"); + printf(" or about 75%% of memory in MB on machines with less\n"); + printf(" is a location where a 19GB file can be saved.\n"); + printf("\n"); + printf("A typical command to update a database imported with the above command is\n"); + printf(" osmosis --rri workingDirectory= --simc --wx - \\\n"); + printf(" | %s -a -d gis --slim -k --flat-nodes \n", name); + printf("where\n"); + printf(" is the same location as above.\n"); + printf(" is the location osmosis replication was initialized to.\n"); + printf("\nRun %s --help --verbose (-h -v) for a full list of options.\n", name); + } + + } + +std::string build_conninfo(const std::string &db, + const boost::optional &username, + const boost::optional &password, + const boost::optional &host, + const std::string &port) +{ + std::ostringstream out; + + out << "dbname='" << db << "'"; + + if (username) { + out << " user='" << *username << "'"; + } + if (password) { + out << " password='" << *password << "'"; + } + if (host) { + out << " host='" << *host << "'"; + } + out << " port='" << port << "'"; + + return out.str(); +} +} // anonymous namespace + + +options_t::options_t(): + conninfo(""), prefix("planet_osm"), scale(DEFAULT_SCALE), projection(new reprojection(PROJ_SPHERE_MERC)), append(0), slim(0), + cache(800), tblsmain_index(boost::none), tblsslim_index(boost::none), tblsmain_data(boost::none), tblsslim_data(boost::none), style(OSM2PGSQL_DATADIR "/default.style"), + expire_tiles_zoom(-1), expire_tiles_zoom_min(-1), expire_tiles_filename("dirty_tiles"), hstore_mode(HSTORE_NONE), enable_hstore_index(0), + enable_multi(false), hstore_columns(), keep_coastlines(0), parallel_indexing(1), + #ifdef __amd64__ + alloc_chunkwise(ALLOC_SPARSE | ALLOC_DENSE), + #else + alloc_chunkwise(ALLOC_SPARSE), + #endif + num_procs(1), droptemp(0), unlogged(0), hstore_match_only(0), flat_node_cache_enabled(0), excludepoly(0), flat_node_file(boost::none), + tag_transform_script(boost::none), tag_transform_node_func(boost::none), tag_transform_way_func(boost::none), + tag_transform_rel_func(boost::none), tag_transform_rel_mem_func(boost::none), + create(0), sanitize(0), long_usage_bool(0), pass_prompt(0), db("gis"), username(boost::none), host(boost::none), + password(boost::none), port("5432"), output_backend("pgsql"), input_reader("auto"), bbox(boost::none), extra_attributes(0), verbose(0) +{ + +} + +options_t::~options_t() +{ +} + +options_t options_t::parse(int argc, char *argv[]) +{ + options_t options; + const char *temparg; + int c; + + //keep going while there are args left to handle + // note: optind would seem to need to be set to 1, but that gives valgrind + // errors - setting it to zero seems to work, though. see + // http://stackoverflow.com/questions/15179963/is-it-possible-to-repeat-getopt#15179990 + optind = 0; + while(-1 != (c = getopt_long(argc, argv, short_options, long_options, NULL))) { + + //handle the current arg + switch (c) { + case 'a': + options.append = 1; + break; + case 'b': + options.bbox = optarg; + break; + case 'c': + options.create = 1; + break; + case 'v': + options.verbose = 1; + break; + case 's': + options.slim = 1; + break; + case 'K': + options.keep_coastlines = 1; + break; + case 'u': + options.sanitize = 1; + break; + case 'l': + options.projection.reset(new reprojection(PROJ_LATLONG)); + break; + case 'm': + options.projection.reset(new reprojection(PROJ_SPHERE_MERC)); + break; + case 'E': + options.projection.reset(new reprojection(-atoi(optarg))); + break; + case 'p': + options.prefix = optarg; + break; + case 'd': + options.db = optarg; + break; + case 'C': + options.cache = atoi(optarg); + break; + case 'U': + options.username = optarg; + break; + case 'W': + options.pass_prompt = 1; + break; + case 'H': + options.host = optarg; + break; + case 'P': + options.port = optarg; + break; + case 'S': + options.style = optarg; + break; + case 'i': + options.tblsmain_index = options.tblsslim_index = optarg; + break; + case 200: + options.tblsslim_data = optarg; + break; + case 201: + options.tblsslim_index = optarg; + break; + case 202: + options.tblsmain_data = optarg; + break; + case 203: + options.tblsmain_index = optarg; + break; + case 'e': + options.expire_tiles_zoom_min = atoi(optarg); + temparg = strchr(optarg, '-'); + if (temparg) + options.expire_tiles_zoom = atoi(temparg + 1); + if (options.expire_tiles_zoom < options.expire_tiles_zoom_min) + options.expire_tiles_zoom = options.expire_tiles_zoom_min; + break; + case 'o': + options.expire_tiles_filename = optarg; + break; + case 'O': + options.output_backend = optarg; + break; + case 'x': + options.extra_attributes = 1; + break; + case 'k': + if (options.hstore_mode != HSTORE_NONE) { + throw std::runtime_error("You can not specify both --hstore (-k) and --hstore-all (-j)\n"); + } + options.hstore_mode = HSTORE_NORM; + break; + case 208: + options.hstore_match_only = 1; + break; + case 'j': + if (options.hstore_mode != HSTORE_NONE) { + throw std::runtime_error("You can not specify both --hstore (-k) and --hstore-all (-j)\n"); + } + options.hstore_mode = HSTORE_ALL; + break; + case 'z': + options.hstore_columns.push_back(optarg); + break; + case 'G': + options.enable_multi = true; + break; + case 'r': + options.input_reader = optarg; + break; + case 'h': + options.long_usage_bool = 1; + break; + case 'I': +#ifdef HAVE_PTHREAD + options.parallel_indexing = 0; +#endif + break; + case 204: + if (strcmp(optarg, "dense") == 0) + options.alloc_chunkwise = ALLOC_DENSE; + else if (strcmp(optarg, "chunk") == 0) + options.alloc_chunkwise = ALLOC_DENSE | ALLOC_DENSE_CHUNK; + else if (strcmp(optarg, "sparse") == 0) + options.alloc_chunkwise = ALLOC_SPARSE; + else if (strcmp(optarg, "optimized") == 0) + options.alloc_chunkwise = ALLOC_DENSE | ALLOC_SPARSE; + else { + throw std::runtime_error((boost::format("Unrecognized cache strategy %1%.\n") % optarg).str()); + } + break; + case 205: +#ifdef HAVE_FORK + options.num_procs = atoi(optarg); +#else + fprintf(stderr, "WARNING: osm2pgsql was compiled without fork, only using one process!\n"); +#endif + break; + case 206: + options.droptemp = 1; + break; + case 207: + options.unlogged = 1; + break; + case 209: + options.flat_node_cache_enabled = 1; + options.flat_node_file = optarg; + break; + case 210: + options.excludepoly = 1; + break; + case 211: + options.enable_hstore_index = 1; + break; + case 212: + options.tag_transform_script = optarg; + break; + case 'V': + exit (EXIT_SUCCESS); + break; + case '?': + default: + short_usage(argv[0]); + break; + } + } //end while + + //they were looking for usage info + if (options.long_usage_bool) { + long_usage(argv[0], options.verbose); + } + + //we require some input files! + if (argc == optind) { + short_usage(argv[0]); + } + + //get the input files + while (optind < argc) { + options.input_files.push_back(std::string(argv[optind])); + optind++; + } + + if (options.append && options.create) { + throw std::runtime_error("--append and --create options can not be used at the same time!\n"); + } + + if (options.append && !options.slim) { + throw std::runtime_error("--append can only be used with slim mode!\n"); + } + + if (options.droptemp && !options.slim) { + throw std::runtime_error("--drop only makes sense with --slim.\n"); + } + + if (options.unlogged && !options.create) { + fprintf(stderr, "Warning: --unlogged only makes sense with --create; ignored.\n"); + options.unlogged = 0; + } + + if (options.hstore_mode == HSTORE_NONE && options.hstore_columns.size() == 0 && options.hstore_match_only) { + fprintf(stderr, "Warning: --hstore-match-only only makes sense with --hstore, --hstore-all, or --hstore-column; ignored.\n"); + options.hstore_match_only = 0; + } + + if (options.enable_hstore_index && options.hstore_mode == HSTORE_NONE && options.hstore_columns.size() == 0) { + fprintf(stderr, "Warning: --hstore-add-index only makes sense with hstore enabled.\n"); + options.enable_hstore_index = 0; + } + + if (options.cache < 0) + options.cache = 0; + + if (options.cache == 0) { + fprintf(stderr, "WARNING: ram cache is disabled. This will likely slow down processing a lot.\n\n"); + } + if (sizeof(int*) == 4 && options.slim != 1) { + fprintf(stderr, "\n!! You are running this on 32bit system, so at most\n"); + fprintf(stderr, "!! 3GB of RAM can be used. If you encounter unexpected\n"); + fprintf(stderr, "!! exceptions during import, you should try running in slim\n"); + fprintf(stderr, "!! mode using parameter -s.\n"); + } + + if (options.pass_prompt) { + char *prompt = simple_prompt("Password:", 100, 0); + if (prompt == NULL) { + options.password = boost::none; + } else { + options.password = std::string(prompt); + } + } else { + char *pgpass = getenv("PGPASS"); + if (pgpass == NULL) { + options.password = boost::none; + } else { + options.password = std::string(pgpass); + } + } + + if (options.num_procs < 1) + options.num_procs = 1; + + //NOTE: this is hugely important if you set it inappropriately and are are caching nodes + //you could get overflow when working with larger coordinates (mercator) and larger scales + options.scale = (options.projection->get_proj_id() == PROJ_LATLONG) ? 10000000 : 100; + options.conninfo = build_conninfo(options.db, options.username, options.password, options.host, options.port); + + return options; +} diff -Nru osm2pgsql-0.82.0/options.hpp osm2pgsql-0.88.1/options.hpp --- osm2pgsql-0.82.0/options.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/options.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,87 @@ +#ifndef OPTION_H +#define OPTION_H + +#include "node-ram-cache.hpp" +#include "reprojection.hpp" + +#include +#include +#include +#include + +/* Variants for generation of hstore column */ +/* No hstore column */ +#define HSTORE_NONE 0 +/* create a hstore column for all tags which do not have an exclusive column */ +#define HSTORE_NORM 1 +/* create a hstore column for all tags */ +#define HSTORE_ALL 2 + +/* Scale is chosen such that 40,000 * SCALE < 2^32 */ +enum { DEFAULT_SCALE = 100 }; + +//TODO: GO THROUGH AND UPDATE TO BOOL WHERE MEMBER DENOTES ONLY ON OR OFF OPTION +struct options_t { +public: + /* construct with sensible defaults */ + options_t(); + virtual ~options_t(); + + static options_t parse(int argc, char *argv[]); + + std::string conninfo; /* Connection info string */ + std::string prefix; /* prefix for table names */ + int scale; /* scale for converting coordinates to fixed point */ + boost::shared_ptr projection; /* SRS of projection */ + bool append; /* Append to existing data */ + bool slim; /* In slim mode */ + int cache; /* Memory usable for cache in MB */ + boost::optional tblsmain_index; /* Pg Tablespace to store indexes on main tables (no default TABLESPACE)*/ + boost::optional tblsslim_index; /* Pg Tablespace to store indexes on slim tables (no default TABLESPACE)*/ + boost::optional tblsmain_data; /* Pg Tablespace to store main tables (no default TABLESPACE)*/ + boost::optional tblsslim_data; /* Pg Tablespace to store slim tables (no default TABLESPACE)*/ + std::string style; /* style file to use */ + int expire_tiles_zoom; /* Zoom level for tile expiry list */ + int expire_tiles_zoom_min; /* Minimum zoom level for tile expiry list */ + std::string expire_tiles_filename; /* File name to output expired tiles list to */ + int hstore_mode; /* add an additional hstore column with objects key/value pairs */ + int enable_hstore_index; /* add an index on the hstore column */ + bool enable_multi; /* Output multi-geometries intead of several simple geometries */ + std::vector hstore_columns; /* list of columns that should be written into their own hstore column */ + int keep_coastlines; + int parallel_indexing; + int alloc_chunkwise; + int num_procs; + int droptemp; /* drop slim mode temp tables after act */ + int unlogged; /* use unlogged tables where possible */ + int hstore_match_only; /* only copy rows that match an explicitly listed key */ + int flat_node_cache_enabled; + int excludepoly; + boost::optional flat_node_file; + boost::optional tag_transform_script, + tag_transform_node_func, // these options allow you to control the name of the + tag_transform_way_func, // Lua functions which get called in the tag transform + tag_transform_rel_func, // script. this is mostly useful in with the "multi" + tag_transform_rel_mem_func; // output so that a single script file can be used. + + int create; + int sanitize; + int long_usage_bool; + int pass_prompt; + std::string db; + boost::optional username; + boost::optional host; + boost::optional password; + std::string port; + std::string output_backend ; + std::string input_reader; + boost::optional bbox; + int extra_attributes; + int verbose; + + std::vector input_files; +private: + +}; + +#endif diff -Nru osm2pgsql-0.82.0/osm2pgsql.c osm2pgsql-0.88.1/osm2pgsql.c --- osm2pgsql-0.82.0/osm2pgsql.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/osm2pgsql.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,754 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -# Use: osm2pgsql planet.osm.bz2 -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#include "config.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -#include "osmtypes.h" -#include "build_geometry.h" -#include "middle-pgsql.h" -#include "middle-ram.h" -#include "node-ram-cache.h" -#include "output-pgsql.h" -#include "output-gazetteer.h" -#include "output-null.h" -#include "sanitizer.h" -#include "reprojection.h" -#include "text-tree.h" -#include "input.h" -#include "sprompt.h" -#include "parse-xml2.h" -#include "parse-primitive.h" -#include "parse-o5m.h" - -#ifdef BUILD_READER_PBF -# include "parse-pbf.h" -#endif - -#define INIT_MAX_MEMBERS 64 -#define INIT_MAX_NODES 4096 - -int verbose; - -/* Data structure carrying all parsing related variables */ -static struct osmdata_t osmdata = { - .filetype = FILETYPE_NONE, - .action = ACTION_NONE, - .bbox = NULL -}; - - -static int parse_bbox(struct osmdata_t *osmdata) -{ - int n; - - if (!osmdata->bbox) - return 0; - - n = sscanf(osmdata->bbox, "%lf,%lf,%lf,%lf", &(osmdata->minlon), &(osmdata->minlat), &(osmdata->maxlon), &(osmdata->maxlat)); - if (n != 4) { - fprintf(stderr, "Bounding box must be specified like: minlon,minlat,maxlon,maxlat\n"); - return 1; - } - if (osmdata->maxlon <= osmdata->minlon) { - fprintf(stderr, "Bounding box failed due to maxlon <= minlon\n"); - return 1; - } - if (osmdata->maxlat <= osmdata->minlat) { - fprintf(stderr, "Bounding box failed due to maxlat <= minlat\n"); - return 1; - } - fprintf(stderr, "Applying Bounding box: %f,%f to %f,%f\n", osmdata->minlon, osmdata->minlat, osmdata->maxlon, osmdata->maxlat); - return 0; -} - - - -void exit_nicely() -{ - fprintf(stderr, "Error occurred, cleaning up\n"); - osmdata.out->cleanup(); - exit(1); -} - -static void short_usage(char *arg0) -{ - const char *name = basename(arg0); - - fprintf(stderr, "Usage error. For further information see:\n"); - fprintf(stderr, "\t%s -h|--help\n", name); -} - -static void long_usage(char *arg0) -{ - int i; - const char *name = basename(arg0); - - printf("Usage:\n"); - printf("\t%s [options] planet.osm\n", name); - printf("\t%s [options] planet.osm.{gz,bz2}\n", name); - printf("\t%s [options] file1.osm file2.osm file3.osm\n", name); - printf("\nThis will import the data from the OSM file(s) into a PostgreSQL database\n"); - printf("suitable for use by the Mapnik renderer\n"); - printf("\nOptions:\n"); - printf(" -a|--append\t\tAdd the OSM file into the database without removing\n"); - printf(" \t\texisting data.\n"); - printf(" -b|--bbox\t\tApply a bounding box filter on the imported data\n"); - printf(" \t\tMust be specified as: minlon,minlat,maxlon,maxlat\n"); - printf(" \t\te.g. --bbox -0.5,51.25,0.5,51.75\n"); - printf(" -c|--create\t\tRemove existing data from the database. This is the \n"); - printf(" \t\tdefault if --append is not specified.\n"); - printf(" -d|--database\tThe name of the PostgreSQL database to connect\n"); - printf(" \t\tto (default: gis).\n"); - printf(" -i|--tablespace-index\tThe name of the PostgreSQL tablespace where\n"); - printf(" \t\tall indexes will be created.\n"); - printf(" \t\tThe following options allow more fine-grained control:\n"); - printf(" --tablespace-main-data \ttablespace for main tables\n"); - printf(" --tablespace-main-index\ttablespace for main table indexes\n"); - printf(" --tablespace-slim-data \ttablespace for slim mode tables\n"); - printf(" --tablespace-slim-index\ttablespace for slim mode indexes\n"); - printf(" \t\t(if unset, use db's default; -i is equivalent to setting\n"); - printf(" \t\t--tablespace-main-index and --tablespace-slim-index)\n"); - printf(" -l|--latlong\t\tStore data in degrees of latitude & longitude.\n"); - printf(" -m|--merc\t\tStore data in proper spherical mercator (default)\n"); - printf(" -M|--oldmerc\t\tStore data in the legacy OSM mercator format\n"); - printf(" -E|--proj num\tUse projection EPSG:num\n"); - printf(" -u|--utf8-sanitize\tRepair bad UTF8 input data (present in planet\n"); - printf(" \tdumps prior to August 2007). Adds about 10%% overhead.\n"); - printf(" -p|--prefix\t\tPrefix for table names (default planet_osm)\n"); - printf(" -s|--slim\t\tStore temporary data in the database. This greatly\n"); - printf(" \t\treduces the RAM usage but is much slower. This switch is\n"); - printf(" \t\trequired if you want to update with --append later.\n"); - - if (sizeof(int*) == 4) { - printf(" \t\tThis program was compiled on a 32bit system, so at most\n"); - printf(" \t\t3GB of RAM will be used. If you encounter problems\n"); - printf(" \t\tduring import, you should try this switch.\n"); - } - printf(" --drop\t\tonly with --slim: drop temporary tables after import (no updates).\n"); - - printf(" -S|--style\t\tLocation of the style file. Defaults to " OSM2PGSQL_DATADIR "/default.style\n"); - printf(" -C|--cache\t\tNow required for slim and non-slim modes: \n"); - printf(" \t\tUse up to this many MB for caching nodes (default: 800)\n"); - printf(" -U|--username\tPostgresql user name\n"); - printf(" \t\tpassword can be given by prompt or PGPASS environment variable.\n"); - printf(" -W|--password\tForce password prompt.\n"); - printf(" -H|--host\t\tDatabase server hostname or socket location.\n"); - printf(" -P|--port\t\tDatabase server port.\n"); - printf(" -e|--expire-tiles [min_zoom-]max_zoom\tCreate a tile expiry list.\n"); - printf(" -o|--expire-output filename\tOutput filename for expired tiles list.\n"); - printf(" -r|--input-reader\tInput frontend.\n"); - printf(" \t\tlibxml2 - Parse XML using libxml2. (default)\n"); - printf(" \t\tprimitive - Primitive XML parsing.\n"); -#ifdef BUILD_READER_PBF - printf(" \t\tpbf - OSM binary format.\n"); -#endif - printf(" -O|--output\t\tOutput backend.\n"); - printf(" \t\tpgsql - Output to a PostGIS database. (default)\n"); - printf(" \t\tgazetteer - Output to a PostGIS database suitable for gazetteer\n"); - printf(" \t\tnull - No output. Useful for testing.\n"); - printf(" -x|--extra-attributes\n"); - printf(" \t\tInclude attributes for each object in the database.\n"); - printf(" \t\tThis includes the username, userid, timestamp and version.\n"); - printf(" \t\tNote: this option also requires additional entries in your style file.\n"); - printf(" -k|--hstore\t\tAdd tags without column to an additional hstore (key/value) column to postgresql tables\n"); - printf(" --hstore-match-only\tOnly keep objects that have a value in one of the columns\n"); - printf(" - \t(normal action with --hstore is to keep all objects)\n"); - printf(" -j|--hstore-all\tAdd all tags to an additional hstore (key/value) column in postgresql tables\n"); - printf(" -z|--hstore-column\tAdd an additional hstore (key/value) column containing all tags\n"); - printf(" \tthat start with the specified string, eg --hstore-column \"name:\" will\n"); - printf(" \tproduce an extra hstore column that contains all name:xx tags\n"); - printf(" --hstore-add-index\tAdd index to hstore column.\n"); - printf(" -G|--multi-geometry\tGenerate multi-geometry features in postgresql tables.\n"); - printf(" -K|--keep-coastlines\tKeep coastline data rather than filtering it out.\n"); - printf(" \t\tBy default natural=coastline tagged data will be discarded based on the\n"); - printf(" \t\tassumption that post-processed Coastline Checker shapefiles will be used.\n"); - printf(" --exclude-invalid-polygon\n"); -#ifdef HAVE_FORK - printf(" --number-processes\t\tSpecifies the number of parallel processes used for certain operations\n"); - printf(" \t\tDefault is 1\n"); -#endif - printf(" -I|--disable-parallel-indexing\tDisable indexing all tables concurrently.\n"); - printf(" --unlogged\tUse unlogged tables (lost on crash but faster). Requires PostgreSQL 9.1.\n"); - printf(" --cache-strategy\tSpecifies the method used to cache nodes in ram.\n"); - printf(" \t\tAvailable options are:\n"); - printf(" \t\tdense: caching strategy optimised for full planet import\n"); - printf(" \t\tchunked: caching strategy optimised for non-contigouse memory allocation\n"); - printf(" \t\tsparse: caching strategy optimised for small extracts\n"); - printf(" \t\toptimized: automatically combines dense and sparse strategies for optimal storage efficiency.\n"); - printf(" \t\t optimized may use twice as much virtual memory, but no more physical memory\n"); - -#ifdef __amd64__ - printf(" \t\t The default is \"optimized\"\n"); -#else - /* use "chunked" as a default in 32 bit compilations, as it is less wasteful of virtual memory than "optimized"*/ - printf(" \t\t The default is \"sparse\"\n"); -#endif - printf(" --flat-nodes\tSpecifies the flat file to use to persistently store node information in slim mode instead of in pgsql\n"); - printf(" \t\tThis file is a single > 16Gb large file. This method is only recomended for full planet imports\n"); - printf(" \t\tas it doesn't work well with small extracts. The default is disabled\n"); - printf(" -h|--help\t\tHelp information.\n"); - printf(" -v|--verbose\t\tVerbose output.\n"); - printf("\n"); - if(!verbose) - { - printf("Add -v to display supported projections.\n"); - printf("Use -E to access any espg projections (usually in /usr/share/proj/epsg)\n" ); - } - else - { - printf("Supported projections:\n" ); - for(i=0; ind_max == 0 ) - osmdata->nd_max = INIT_MAX_NODES; - else - osmdata->nd_max <<= 1; - - osmdata->nds = realloc( osmdata->nds, osmdata->nd_max * sizeof( osmdata->nds[0] ) ); - if( !osmdata->nds ) - { - fprintf( stderr, "Failed to expand node list to %d\n", osmdata->nd_max ); - exit_nicely(); - } -} - -void realloc_members(struct osmdata_t *osmdata) -{ - if( osmdata->member_max == 0 ) - osmdata->member_max = INIT_MAX_NODES; - else - osmdata->member_max <<= 1; - - osmdata->members = realloc( osmdata->members, osmdata->member_max * sizeof( osmdata->members[0] ) ); - if( !osmdata->members ) - { - fprintf( stderr, "Failed to expand member list to %d\n", osmdata->member_max ); - exit_nicely(); - } -} - -void resetMembers(struct osmdata_t *osmdata) -{ - unsigned i; - for(i = 0; i < osmdata->member_count; i++ ) - free( osmdata->members[i].role ); -} - -void printStatus(struct osmdata_t *osmdata) -{ - time_t now; - time_t end_nodes; - time_t end_way; - time_t end_rel; - time(&now); - end_nodes = osmdata->start_way > 0 ? osmdata->start_way : now; - end_way = osmdata->start_rel > 0 ? osmdata->start_rel : now; - end_rel = now; - fprintf(stderr, "\rProcessing: Node(%" PRIdOSMID "k %.1fk/s) Way(%" PRIdOSMID "k %.2fk/s) Relation(%" PRIdOSMID " %.2f/s)", - osmdata->count_node/1000, - (double)osmdata->count_node/1000.0/((int)(end_nodes - osmdata->start_node) > 0 ? (double)(end_nodes - osmdata->start_node) : 1.0), - osmdata->count_way/1000, - osmdata->count_way > 0 ? (double)osmdata->count_way/1000.0/ - ((double)(end_way - osmdata->start_way) > 0.0 ? (double)(end_way - osmdata->start_way) : 1.0) : 0.0, - osmdata->count_rel, - osmdata->count_rel > 0 ? (double)osmdata->count_rel/ - ((double)(end_rel - osmdata->start_rel) > 0.0 ? (double)(end_rel - osmdata->start_rel) : 1.0) : 0.0); -} - -int node_wanted(struct osmdata_t *osmdata, double lat, double lon) -{ - if (!osmdata->bbox) - return 1; - - if (lat < osmdata->minlat || lat > osmdata->maxlat) - return 0; - if (lon < osmdata->minlon || lon > osmdata->maxlon) - return 0; - return 1; -} - -int main(int argc, char *argv[]) -{ - int append=0; - int create=0; - int slim=0; - int sanitize=0; - int long_usage_bool=0; - int pass_prompt=0; - int projection = PROJ_SPHERE_MERC; - int expire_tiles_zoom = -1; - int expire_tiles_zoom_min = -1; - int enable_hstore = HSTORE_NONE; - int enable_hstore_index = 0; - int hstore_match_only = 0; - int enable_multi = 0; - int parallel_indexing = 1; - int flat_node_cache_enabled = 0; -#ifdef __amd64__ - int alloc_chunkwise = ALLOC_SPARSE | ALLOC_DENSE; -#else - int alloc_chunkwise = ALLOC_SPARSE; -#endif - int num_procs = 1; - int droptemp = 0; - int unlogged = 0; - int excludepoly = 0; - time_t start, end; - time_t overall_start, overall_end; - time_t now; - time_t end_nodes; - time_t end_way; - time_t end_rel; - const char *expire_tiles_filename = "dirty_tiles"; - const char *db = "gis"; - const char *username=NULL; - const char *host=NULL; - const char *password=NULL; - const char *port = "5432"; - const char *tblsmain_index = NULL; /* no default TABLESPACE for index on main tables */ - const char *tblsmain_data = NULL; /* no default TABLESPACE for main tables */ - const char *tblsslim_index = NULL; /* no default TABLESPACE for index on slim mode tables */ - const char *tblsslim_data = NULL; /* no default TABLESPACE for slim mode tables */ - const char *conninfo = NULL; - const char *prefix = "planet_osm"; - const char *style = OSM2PGSQL_DATADIR "/default.style"; - const char *temparg; - const char *output_backend = "pgsql"; - const char *input_reader = "auto"; - const char **hstore_columns = NULL; - const char *flat_nodes_file = NULL; - int n_hstore_columns = 0; - int keep_coastlines=0; - int cache = 800; - struct output_options options; - PGconn *sql_conn; - - int (*streamFile)(char *, int, struct osmdata_t *); - - fprintf(stderr, "osm2pgsql SVN version %s (%lubit id space)\n\n", VERSION, 8 * sizeof(osmid_t)); - - while (1) { - int c, option_index = 0; - static struct option long_options[] = { - {"append", 0, 0, 'a'}, - {"bbox", 1, 0, 'b'}, - {"create", 0, 0, 'c'}, - {"database", 1, 0, 'd'}, - {"latlong", 0, 0, 'l'}, - {"verbose", 0, 0, 'v'}, - {"slim", 0, 0, 's'}, - {"prefix", 1, 0, 'p'}, - {"proj", 1, 0, 'E'}, - {"merc", 0, 0, 'm'}, - {"oldmerc", 0, 0, 'M'}, - {"utf8-sanitize", 0, 0, 'u'}, - {"cache", 1, 0, 'C'}, - {"username", 1, 0, 'U'}, - {"password", 0, 0, 'W'}, - {"host", 1, 0, 'H'}, - {"port", 1, 0, 'P'}, - {"tablespace-index", 1, 0, 'i'}, - {"tablespace-slim-data", 1, 0, 200}, - {"tablespace-slim-index", 1, 0, 201}, - {"tablespace-main-data", 1, 0, 202}, - {"tablespace-main-index", 1, 0, 203}, - {"help", 0, 0, 'h'}, - {"style", 1, 0, 'S'}, - {"expire-tiles", 1, 0, 'e'}, - {"expire-output", 1, 0, 'o'}, - {"output", 1, 0, 'O'}, - {"extra-attributes", 0, 0, 'x'}, - {"hstore", 0, 0, 'k'}, - {"hstore-all", 0, 0, 'j'}, - {"hstore-column", 1, 0, 'z'}, - {"hstore-match-only", 0, 0, 208}, - {"hstore-add-index",0,0,211}, - {"multi-geometry", 0, 0, 'G'}, - {"keep-coastlines", 0, 0, 'K'}, - {"input-reader", 1, 0, 'r'}, - {"version", 0, 0, 'V'}, - {"disable-parallel-indexing", 0, 0, 'I'}, - {"cache-strategy", 1, 0, 204}, - {"number-processes", 1, 0, 205}, - {"drop", 0, 0, 206}, - {"unlogged", 0, 0, 207}, - {"flat-nodes",1,0,209}, - {"exclude-invalid-polygon",0,0,210}, - {0, 0, 0, 0} - }; - - c = getopt_long (argc, argv, "ab:cd:KhlmMp:suvU:WH:P:i:IE:C:S:e:o:O:xkjGz:r:V", long_options, &option_index); - if (c == -1) - break; - - switch (c) { - case 'a': append=1; break; - case 'b': osmdata.bbox=optarg; break; - case 'c': create=1; break; - case 'v': verbose=1; break; - case 's': slim=1; break; - case 'K': keep_coastlines=1; break; - case 'u': sanitize=1; break; - case 'l': projection=PROJ_LATLONG; break; - case 'm': projection=PROJ_SPHERE_MERC; break; - case 'M': projection=PROJ_MERC; break; - case 'E': projection=-atoi(optarg); break; - case 'p': prefix=optarg; break; - case 'd': db=optarg; break; - case 'C': cache = atoi(optarg); break; - case 'U': username=optarg; break; - case 'W': pass_prompt=1; break; - case 'H': host=optarg; break; - case 'P': port=optarg; break; - case 'S': style=optarg; break; - case 'i': tblsmain_index=tblsslim_index=optarg; break; - case 200: tblsslim_data=optarg; break; - case 201: tblsslim_index=optarg; break; - case 202: tblsmain_data=optarg; break; - case 203: tblsmain_index=optarg; break; - case 'e': - expire_tiles_zoom_min = atoi(optarg); - temparg = strchr(optarg, '-'); - if (temparg) expire_tiles_zoom = atoi(temparg + 1); - if (expire_tiles_zoom < expire_tiles_zoom_min) expire_tiles_zoom = expire_tiles_zoom_min; - break; - case 'o': expire_tiles_filename=optarg; break; - case 'O': output_backend = optarg; break; - case 'x': osmdata.extra_attributes=1; break; - case 'k': enable_hstore=HSTORE_NORM; break; - case 208: hstore_match_only = 1; break; - case 'j': enable_hstore=HSTORE_ALL; break; - case 'z': - n_hstore_columns++; - hstore_columns = (const char**)realloc(hstore_columns, sizeof(char *) * n_hstore_columns); - hstore_columns[n_hstore_columns-1] = optarg; - break; - case 'G': enable_multi=1; break; - case 'r': input_reader = optarg; break; - case 'h': long_usage_bool=1; break; - case 'I': -#ifdef HAVE_PTHREAD - parallel_indexing=0; -#endif - break; - case 204: - if (strcmp(optarg,"dense") == 0) alloc_chunkwise = ALLOC_DENSE; - if (strcmp(optarg,"chunk") == 0) alloc_chunkwise = ALLOC_DENSE | ALLOC_DENSE_CHUNK; - if (strcmp(optarg,"sparse") == 0) alloc_chunkwise = ALLOC_SPARSE; - if (strcmp(optarg,"optimized") == 0) alloc_chunkwise = ALLOC_DENSE | ALLOC_SPARSE; - break; - case 205: -#ifdef HAVE_FORK - num_procs = atoi(optarg); -#else - fprintf(stderr, "WARNING: osm2pgsql was compiled without fork, only using one process!\n"); -#endif - break; - case 206: droptemp = 1; break; - case 207: unlogged = 1; break; - case 209: - flat_node_cache_enabled = 1; - flat_nodes_file = optarg; - break; - case 210: excludepoly = 1; exclude_broken_polygon(); break; - case 211: enable_hstore_index = 1; break; - case 'V': exit(EXIT_SUCCESS); - case '?': - default: - short_usage(argv[0]); - exit(EXIT_FAILURE); - } - } - - if (long_usage_bool) { - long_usage(argv[0]); - exit(EXIT_SUCCESS); - } - - if (argc == optind) { /* No non-switch arguments */ - short_usage(argv[0]); - exit(EXIT_FAILURE); - } - - if (append && create) { - fprintf(stderr, "Error: --append and --create options can not be used at the same time!\n"); - exit(EXIT_FAILURE); - } - - if (droptemp && !slim) { - fprintf(stderr, "Error: --drop only makes sense with --slim.\n"); - exit(EXIT_FAILURE); - } - - if (unlogged && !create) { - fprintf(stderr, "Warning: --unlogged only makes sense with --create; ignored.\n"); - unlogged = 0; - } - - if (enable_hstore == HSTORE_NONE && !n_hstore_columns && hstore_match_only) - { - fprintf(stderr, "Warning: --hstore-match-only only makes sense with --hstore, --hstore-all, or --hstore-column; ignored.\n"); - hstore_match_only = 0; - } - - if (enable_hstore_index && enable_hstore == HSTORE_NONE && !n_hstore_columns) { - fprintf(stderr, "Warning: --hstore-add-index only makes sense with hstore enabled.\n"); - enable_hstore_index = 0; - } - - if (cache < 0) cache = 0; - - if (num_procs < 1) num_procs = 1; - - if (pass_prompt) - password = simple_prompt("Password:", 100, 0); - else { - password = getenv("PGPASS"); - } - - - - conninfo = build_conninfo(db, username, password, host, port); - sql_conn = PQconnectdb(conninfo); - if (PQstatus(sql_conn) != CONNECTION_OK) { - fprintf(stderr, "Error: Connection to database failed: %s\n", PQerrorMessage(sql_conn)); - exit(EXIT_FAILURE); - } - if (unlogged && PQserverVersion(sql_conn) < 90100) { - fprintf(stderr, "Error: --unlogged works only with PostgreSQL 9.1 and above, but\n"); - fprintf(stderr, "you are using PostgreSQL %d.%d.%d.\n", PQserverVersion(sql_conn) / 10000, (PQserverVersion(sql_conn) / 100) % 100, PQserverVersion(sql_conn) % 100); - exit(EXIT_FAILURE); - } - - PQfinish(sql_conn); - - text_init(); - initList(&osmdata.tags); - - osmdata.count_node = osmdata.max_node = 0; - osmdata.count_way = osmdata.max_way = 0; - osmdata.count_rel = osmdata.max_rel = 0; - - LIBXML_TEST_VERSION - - project_init(projection); - fprintf(stderr, "Using projection SRS %d (%s)\n", - project_getprojinfo()->srs, project_getprojinfo()->descr ); - - if (parse_bbox(&osmdata)) - return 1; - - options.conninfo = conninfo; - options.prefix = prefix; - options.append = append; - options.slim = slim; - options.projection = project_getprojinfo()->srs; - options.scale = (projection==PROJ_LATLONG)?10000000:100; - options.mid = slim ? &mid_pgsql : &mid_ram; - options.cache = cache; - options.style = style; - options.tblsmain_index = tblsmain_index; - options.tblsmain_data = tblsmain_data; - options.tblsslim_index = tblsslim_index; - options.tblsslim_data = tblsslim_data; - options.expire_tiles_zoom = expire_tiles_zoom; - options.expire_tiles_zoom_min = expire_tiles_zoom_min; - options.expire_tiles_filename = expire_tiles_filename; - options.enable_multi = enable_multi; - options.enable_hstore = enable_hstore; - options.enable_hstore_index = enable_hstore_index; - options.hstore_match_only = hstore_match_only; - options.hstore_columns = hstore_columns; - options.n_hstore_columns = n_hstore_columns; - options.keep_coastlines = keep_coastlines; - options.parallel_indexing = parallel_indexing; - options.alloc_chunkwise = alloc_chunkwise; - options.num_procs = num_procs; - options.droptemp = droptemp; - options.unlogged = unlogged; - options.flat_node_cache_enabled = flat_node_cache_enabled; - options.flat_node_file = flat_nodes_file; - options.excludepoly = excludepoly; - - if (strcmp("pgsql", output_backend) == 0) { - osmdata.out = &out_pgsql; - } else if (strcmp("gazetteer", output_backend) == 0) { - osmdata.out = &out_gazetteer; - } else if (strcmp("null", output_backend) == 0) { - osmdata.out = &out_null; - } else { - fprintf(stderr, "Output backend `%s' not recognised. Should be one of [pgsql, gazetteer, null].\n", output_backend); - exit(EXIT_FAILURE); - } - options.out = osmdata.out; - - if (strcmp("auto", input_reader) != 0) { - if (strcmp("libxml2", input_reader) == 0) { - streamFile = &streamFileXML2; - } else if (strcmp("primitive", input_reader) == 0) { - streamFile = &streamFilePrimitive; -#ifdef BUILD_READER_PBF - } else if (strcmp("pbf", input_reader) == 0) { - streamFile = &streamFilePbf; -#endif - } else if (strcmp("o5m", input_reader) == 0) { - streamFile = &streamFileO5m; - } else { - fprintf(stderr, "Input parser `%s' not recognised. Should be one of [libxml2, primitive, o5m" -#ifdef BUILD_READER_PBF - ", pbf" -#endif - "].\n", input_reader); - exit(EXIT_FAILURE); - } - } - - time(&overall_start); - osmdata.out->start(&options); - - realloc_nodes(&osmdata); - realloc_members(&osmdata); - - if (sizeof(int*) == 4 && options.slim != 1) { - fprintf(stderr, "\n!! You are running this on 32bit system, so at most\n"); - fprintf(stderr, "!! 3GB of RAM can be used. If you encounter unexpected\n"); - fprintf(stderr, "!! exceptions during import, you should try running in slim\n"); - fprintf(stderr, "!! mode using parameter -s.\n"); - } - - while (optind < argc) { - /* if input_reader is not forced by -r switch try to auto-detect it - by file extension */ - if (strcmp("auto", input_reader) == 0) { - - if (strcasecmp(".pbf",argv[optind]+strlen(argv[optind])-4) == 0) { -#ifdef BUILD_READER_PBF - streamFile = &streamFilePbf; -#else - fprintf(stderr, "ERROR: PBF support has not been compiled into this version of osm2pgsql, please either compile it with pbf support or use one of the other input formats\n"); - exit(EXIT_FAILURE); -#endif - } else if (strcasecmp(".o5m",argv[optind]+strlen(argv[optind])-4) == 0) { - streamFile = &streamFileO5m; - } else { - streamFile = &streamFileXML2; - } - } - fprintf(stderr, "\nReading in file: %s\n", argv[optind]); - time(&start); - if (streamFile(argv[optind], sanitize, &osmdata) != 0) - exit_nicely(); - time(&end); - fprintf(stderr, " parse time: %ds\n", (int)(end - start)); - optind++; - } - - xmlCleanupParser(); - xmlMemoryDump(); - - if (osmdata.count_node || osmdata.count_way || osmdata.count_rel) { - time(&now); - end_nodes = osmdata.start_way > 0 ? osmdata.start_way : now; - end_way = osmdata.start_rel > 0 ? osmdata.start_rel : now; - end_rel = now; - fprintf(stderr, "\n"); - fprintf(stderr, "Node stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_node, osmdata.max_node, - osmdata.count_node > 0 ? (int)(end_nodes - osmdata.start_node) : 0); - fprintf(stderr, "Way stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_way, osmdata.max_way, - osmdata.count_way > 0 ? (int)(end_way - osmdata.start_way) : 0); - fprintf(stderr, "Relation stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", osmdata.count_rel, osmdata.max_rel, - osmdata.count_rel > 0 ? (int)(end_rel - osmdata.start_rel) : 0); - } - osmdata.out->stop(); - - free(osmdata.nds); - free(osmdata.members); - - /* free the column pointer buffer */ - free(hstore_columns); - - project_exit(); - text_exit(); - fprintf(stderr, "\n"); - time(&overall_end); - fprintf(stderr, "Osm2pgsql took %ds overall\n", (int)(overall_end - overall_start)); - - return 0; -} diff -Nru osm2pgsql-0.82.0/osm2pgsql.cpp osm2pgsql-0.88.1/osm2pgsql.cpp --- osm2pgsql-0.82.0/osm2pgsql.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/osm2pgsql.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,131 @@ +/* +#----------------------------------------------------------------------------- +# osm2pgsql - converts planet.osm file into PostgreSQL +# compatible output suitable to be rendered by mapnik +# Use: osm2pgsql planet.osm.bz2 +#----------------------------------------------------------------------------- +# Original Python implementation by Artem Pavlenko +# Re-implementation by Jon Burgess, Copyright 2006 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#include "config.h" +#include "osmtypes.hpp" +#include "reprojection.hpp" +#include "options.hpp" +#include "parse.hpp" +#include "middle.hpp" +#include "output.hpp" +#include "osmdata.hpp" +#include "util.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include + +void check_db(const char* conninfo, const int unlogged) +{ + PGconn *sql_conn = PQconnectdb(conninfo); + + //make sure you can connect + if (PQstatus(sql_conn) != CONNECTION_OK) { + throw std::runtime_error((boost::format("Error: Connection to database failed: %1%\n") % PQerrorMessage(sql_conn)).str()); + } + + //make sure unlogged it is supported by your database if you want it + if (unlogged && PQserverVersion(sql_conn) < 90100) { + throw std::runtime_error(( + boost::format("Error: --unlogged works only with PostgreSQL 9.1 and above, but\n you are using PostgreSQL %1%.%2%.%3%.\n") + % (PQserverVersion(sql_conn) / 10000) + % ((PQserverVersion(sql_conn) / 100) % 100) + % (PQserverVersion(sql_conn) % 100)).str()); + } + + PQfinish(sql_conn); +} + +int main(int argc, char *argv[]) +{ + fprintf(stderr, "osm2pgsql SVN version %s (%lubit id space)\n\n", VERSION, 8 * sizeof(osmid_t)); + try + { + //parse the args into the different options members + options_t options = options_t::parse(argc, argv); + if(options.long_usage_bool) + return 0; + + //setup the front (input) + parse_delegate_t parser(options.extra_attributes, options.bbox, options.projection); + + //setup the middle + boost::shared_ptr middle = middle_t::create_middle(options.slim); + + //setup the backend (output) + std::vector > outputs = output_t::create_outputs(middle.get(), options); + + //let osmdata orchestrate between the middle and the outs + osmdata_t osmdata(middle, outputs); + + //check the database + check_db(options.conninfo.c_str(), options.unlogged); + + fprintf(stderr, "Using projection SRS %d (%s)\n", + options.projection->project_getprojinfo()->srs, + options.projection->project_getprojinfo()->descr ); + + //start it up + time_t overall_start = time(NULL); + osmdata.start(); + + /* Processing + * In this phase the input file(s) are read and parsed, populating some of the + * tables. Not all ways can be handled before relations are processed, so they're + * set as pending, to be handled in the next stage. + */ + //read in the input files one by one + for(std::vector::const_iterator filename = options.input_files.begin(); filename != options.input_files.end(); ++filename) + { + //read the actual input + fprintf(stderr, "\nReading in file: %s\n", filename->c_str()); + time_t start = time(NULL); + if (parser.streamFile(options.input_reader.c_str(), filename->c_str(), options.sanitize, &osmdata) != 0) + util::exit_nicely(); + fprintf(stderr, " parse time: %ds\n", (int)(time(NULL) - start)); + } + + //show stats + parser.printSummary(); + + //Process pending ways, relations, cluster, and create indexes + osmdata.stop(); + + fprintf(stderr, "\nOsm2pgsql took %ds overall\n", (int)(time(NULL) - overall_start)); + + return 0; + }//something went wrong along the way + catch(const std::runtime_error& e) + { + fprintf(stderr, "Osm2pgsql failed due to ERROR: %s\n", e.what()); + exit(EXIT_FAILURE); + } +} diff -Nru osm2pgsql-0.82.0/osm2pgsql-svn.sh osm2pgsql-0.88.1/osm2pgsql-svn.sh --- osm2pgsql-0.82.0/osm2pgsql-svn.sh 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/osm2pgsql-svn.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -#!/bin/sh - -DATE=$(date +%Y%m%d) -MODULE="$(basename $0 -svn.sh)" -SVNROOT=http://svn.openstreetmap.org/applications/utils/export/osm2pgsql/ - -set -x -rm -rf $MODULE - -svn export $SVNROOT $MODULE/ - -## tar it up -tar cjf $MODULE-${DATE}svn.tar.bz2 $MODULE - -## cleanup -rm -rf $MODULE - diff -Nru osm2pgsql-0.82.0/osmdata.cpp osm2pgsql-0.88.1/osmdata.cpp --- osm2pgsql-0.82.0/osmdata.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/osmdata.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,383 @@ +#include "osmdata.hpp" +#include "output.hpp" +#include "middle.hpp" +#include "node-ram-cache.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +osmdata_t::osmdata_t(boost::shared_ptr mid_, const boost::shared_ptr& out_): mid(mid_) +{ + outs.push_back(out_); +} + +osmdata_t::osmdata_t(boost::shared_ptr mid_, const std::vector > &outs_) + : mid(mid_), outs(outs_) +{ + if (outs.empty()) { + throw std::runtime_error("Must have at least one output, but none have " + "been configured."); + } +} + +osmdata_t::~osmdata_t() +{ +} + +int osmdata_t::node_add(osmid_t id, double lat, double lon, const taglist_t &tags) { + mid->nodes_set(id, lat, lon, tags); + + // guarantee that we use the same values as in the node cache + ramNode n(lon, lat); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->node_add(id, n.lat(), n.lon(), tags); + } + return status; +} + +int osmdata_t::way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) { + mid->ways_set(id, nodes, tags); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->way_add(id, nodes, tags); + } + return status; +} + +int osmdata_t::relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) { + mid->relations_set(id, members, tags); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->relation_add(id, members, tags); + } + return status; +} + +int osmdata_t::node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) { + slim_middle_t *slim = dynamic_cast(mid.get()); + + slim->nodes_delete(id); + slim->nodes_set(id, lat, lon, tags); + + // guarantee that we use the same values as in the node cache + ramNode n(lon, lat); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->node_modify(id, n.lat(), n.lon(), tags); + } + + slim->node_changed(id); + + return status; +} + +int osmdata_t::way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) { + slim_middle_t *slim = dynamic_cast(mid.get()); + + slim->ways_delete(id); + slim->ways_set(id, nodes, tags); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->way_modify(id, nodes, tags); + } + + slim->way_changed(id); + + return status; +} + +int osmdata_t::relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) { + slim_middle_t *slim = dynamic_cast(mid.get()); + + slim->relations_delete(id); + slim->relations_set(id, members, tags); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->relation_modify(id, members, tags); + } + + slim->relation_changed(id); + + return status; +} + +int osmdata_t::node_delete(osmid_t id) { + slim_middle_t *slim = dynamic_cast(mid.get()); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->node_delete(id); + } + + slim->nodes_delete(id); + + return status; +} + +int osmdata_t::way_delete(osmid_t id) { + slim_middle_t *slim = dynamic_cast(mid.get()); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->way_delete(id); + } + + slim->ways_delete(id); + + return status; +} + +int osmdata_t::relation_delete(osmid_t id) { + slim_middle_t *slim = dynamic_cast(mid.get()); + + int status = 0; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + status |= out->relation_delete(id); + } + + slim->relations_delete(id); + + return status; +} + +void osmdata_t::start() { + BOOST_FOREACH(boost::shared_ptr& out, outs) { + out->start(); + } + mid->start(outs[0]->get_options()); +} + +namespace { + +//TODO: have the main thread using the main middle to query the middle for batches of ways (configurable number) +//and stuffing those into the work queue, so we have a single producer multi consumer threaded queue +//since the fetching from middle should be faster than the processing in each backend. + +struct pending_threaded_processor : public middle_t::pending_processor { + typedef std::vector > output_vec_t; + typedef std::pair, output_vec_t> clone_t; + + static void do_jobs(output_vec_t const& outputs, pending_queue_t& queue, size_t& ids_done, boost::mutex& mutex, int append, bool ways) { + while (true) { + //get the job off the queue synchronously + pending_job_t job; + mutex.lock(); + if(queue.empty()) { + mutex.unlock(); + break; + } + else { + job = queue.top(); + queue.pop(); + } + mutex.unlock(); + + //process it + if(ways) + outputs.at(job.output_id)->pending_way(job.osm_id, append); + else + outputs.at(job.output_id)->pending_relation(job.osm_id, append); + + mutex.lock(); + ++ids_done; + mutex.unlock(); + } + } + + //starts up count threads and works on the queue + pending_threaded_processor(boost::shared_ptr mid, const output_vec_t& outs, size_t thread_count, size_t job_count, int append) + //note that we cant hint to the stack how large it should be ahead of time + //we could use a different datastructure like a deque or vector but then + //the outputs the enqueue jobs would need the version check for the push(_back) method + : outs(outs), ids_queued(0), append(append), queue(), ids_done(0) { + + //clone all the things we need + clones.reserve(thread_count); + for (size_t i = 0; i < thread_count; ++i) { + //clone the middle + boost::shared_ptr mid_clone = mid->get_instance(); + + //clone the outs + output_vec_t out_clones; + BOOST_FOREACH(const boost::shared_ptr& out, outs) { + out_clones.push_back(out->clone(mid_clone.get())); + } + + //keep the clones for a specific thread to use + clones.push_back(clone_t(mid_clone, out_clones)); + } + } + + ~pending_threaded_processor() {} + + void enqueue_ways(osmid_t id) { + for(size_t i = 0; i < outs.size(); ++i) { + outs[i]->enqueue_ways(queue, id, i, ids_queued); + } + } + + //waits for the completion of all outstanding jobs + void process_ways() { + //reset the number we've done + ids_done = 0; + + fprintf(stderr, "\nGoing over pending ways...\n"); + fprintf(stderr, "\t%zu ways are pending\n", ids_queued); + fprintf(stderr, "\nUsing %zu helper-processes\n", clones.size()); + time_t start = time(NULL); + + + //make the threads and start them + for (size_t i = 0; i < clones.size(); ++i) { + workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, true)); + } + + //TODO: print out partial progress + + //wait for them to really be done + workers.join_all(); + + time_t finish = time(NULL); + fprintf(stderr, "\rFinished processing %zu ways in %i sec\n\n", ids_queued, (int)(finish - start)); + if (finish - start > 0) + fprintf(stderr, "%zu Pending ways took %ds at a rate of %.2f/s\n", ids_queued, (int)(finish - start), + ((double)ids_queued / (double)(finish - start))); + ids_queued = 0; + ids_done = 0; + + //collect all the new rels that became pending from each + //output in each thread back to their respective main outputs + BOOST_FOREACH(const clone_t& clone, clones) { + //for each clone/original output + for(output_vec_t::const_iterator original_output = outs.begin(), clone_output = clone.second.begin(); + original_output != outs.end() && clone_output != clone.second.end(); ++original_output, ++clone_output) { + //done copying ways for now + clone_output->get()->commit(); + //merge the pending from this threads copy of output back + original_output->get()->merge_pending_relations(*clone_output); + } + } + } + + void enqueue_relations(osmid_t id) { + for(size_t i = 0; i < outs.size(); ++i) { + outs[i]->enqueue_relations(queue, id, i, ids_queued); + } + } + + void process_relations() { + //reset the number we've done + ids_done = 0; + + fprintf(stderr, "\nGoing over pending relations...\n"); + fprintf(stderr, "\t%zu relations are pending\n", ids_queued); + fprintf(stderr, "\nUsing %zu helper-processes\n", clones.size()); + time_t start = time(NULL); + + //make the threads and start them + for (size_t i = 0; i < clones.size(); ++i) { + workers.create_thread(boost::bind(do_jobs, boost::cref(clones[i].second), boost::ref(queue), boost::ref(ids_done), boost::ref(mutex), append, false)); + } + + //TODO: print out partial progress + + //wait for them to really be done + workers.join_all(); + + time_t finish = time(NULL); + fprintf(stderr, "\rFinished processing %zu relations in %i sec\n\n", ids_queued, (int)(finish - start)); + if (finish - start > 0) + fprintf(stderr, "%zu Pending relations took %ds at a rate of %.2f/s\n", ids_queued, (int)(finish - start), + ((double)ids_queued / (double)(finish - start))); + ids_queued = 0; + ids_done = 0; + + //collect all expiry tree informations together into one + BOOST_FOREACH(const clone_t& clone, clones) { + //for each clone/original output + for(output_vec_t::const_iterator original_output = outs.begin(), clone_output = clone.second.begin(); + original_output != outs.end() && clone_output != clone.second.end(); ++original_output, ++clone_output) { + //done copying rels for now + clone_output->get()->commit(); + //merge the expire tree from this threads copy of output back + original_output->get()->merge_expire_trees(*clone_output); + } + } + } + +private: + //middle and output copies + std::vector clones; + output_vec_t outs; //would like to move ownership of outs to osmdata_t and middle passed to output_t instead of owned by it + //actual threads + boost::thread_group workers; + //how many jobs do we have in the queue to start with + size_t ids_queued; + //appending to output that is already there (diff processing) + int append; + //job queue + pending_queue_t queue; + + //how many ids within the job have been processed + size_t ids_done; + //so the threads can manage some of the shared state + boost::mutex mutex; +}; + +} // anonymous namespace + +void osmdata_t::stop() { + /* Commit the transactions, so that multiple processes can + * access the data simultanious to process the rest in parallel + * as well as see the newly created tables. + */ + size_t pending_count = mid->pending_count(); + mid->commit(); + BOOST_FOREACH(boost::shared_ptr& out, outs) { + //TODO: each of the outs can be in parallel + out->commit(); + pending_count += out->pending_count(); + } + + // should be the same for all outputs + const int append = outs[0]->get_options()->append; + + //threaded pending processing + pending_threaded_processor ptp(mid, outs, outs[0]->get_options()->num_procs, pending_count, append); + + if (!outs.empty()) { + //This stage takes ways which were processed earlier, but might be + //involved in a multipolygon relation. They could also be ways that + //were modified in diff processing. + mid->iterate_ways( ptp ); + + //This is like pending ways, except there aren't pending relations + //on import, only on update. + //TODO: Can we skip this on import? + mid->iterate_relations( ptp ); + } + + //Clustering, index creation, and cleanup. + //All the intensive parts of this are long-running PostgreSQL commands + boost::thread_group threads; + BOOST_FOREACH(boost::shared_ptr& out, outs) { + threads.add_thread(new boost::thread(boost::bind( &output_t::stop, out.get() ))); + } + threads.add_thread(new boost::thread(boost::bind( &middle_t::stop, mid.get() ))); + threads.join_all(); +} diff -Nru osm2pgsql-0.82.0/osmdata.hpp osm2pgsql-0.88.1/osmdata.hpp --- osm2pgsql-0.82.0/osmdata.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/osmdata.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,42 @@ +#ifndef OSMDATA_H +#define OSMDATA_H + +// when __cplusplus is defined, we need to define this macro as well +// to get the print format specifiers in the inttypes.h header. +#include "config.h" + +#include +#include + +#include "osmtypes.hpp" + +class output_t; +struct middle_t; + +class osmdata_t { +public: + osmdata_t(boost::shared_ptr mid_, const boost::shared_ptr& out_); + osmdata_t(boost::shared_ptr mid_, const std::vector > &outs_); + ~osmdata_t(); + + void start(); + void stop(); + + int node_add(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_delete(osmid_t id); + int way_delete(osmid_t id); + int relation_delete(osmid_t id); + +private: + boost::shared_ptr mid; + std::vector > outs; +}; + +#endif diff -Nru osm2pgsql-0.82.0/osmtypes.h osm2pgsql-0.88.1/osmtypes.h --- osm2pgsql-0.82.0/osmtypes.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/osmtypes.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,82 +0,0 @@ -/* Data types to hold OSM node, segment, way data */ - -#ifndef OSMTYPES_H -#define OSMTYPES_H - -#include -#include -#include - -/* Use ./configure --enable-64bit-ids to build a version that supports 64bit IDs. */ - -#ifdef OSMID64 -typedef int64_t osmid_t; -#define strtoosmid strtoll -#define PRIdOSMID PRId64 -#define POSTGRES_OSMID_TYPE "int8" -#else -typedef int32_t osmid_t; -#define strtoosmid strtol -#define PRIdOSMID PRId32 -#define POSTGRES_OSMID_TYPE "int4" -#endif - -#include "keyvals.h" - -enum OsmType { OSMTYPE_WAY, OSMTYPE_NODE, OSMTYPE_RELATION }; - -struct osmNode { - double lon; - double lat; -}; - -struct member { - enum OsmType type; - osmid_t id; - char *role; -}; - -typedef enum { FILETYPE_NONE, FILETYPE_OSM, FILETYPE_OSMCHANGE, FILETYPE_PLANETDIFF } filetypes_t; -typedef enum { ACTION_NONE, ACTION_CREATE, ACTION_MODIFY, ACTION_DELETE } actions_t; - -struct osmdata_t { - osmid_t count_node, max_node; - osmid_t count_way, max_way; - osmid_t count_rel, max_rel; - time_t start_node, start_way, start_rel; - - struct output_t *out; - -/* Since {node,way} elements are not nested we can guarantee the - values in an end tag must match those of the corresponding - start tag and can therefore be cached. -*/ - double node_lon, node_lat; - struct keyval tags; - osmid_t *nds; - int nd_count, nd_max; - struct member *members; - int member_count, member_max; - osmid_t osm_id; - filetypes_t filetype; - actions_t action; - int extra_attributes; - - /* Bounding box to filter imported data */ - const char *bbox; - - double minlon, minlat, maxlon, maxlat; - - int parallel_indexing; -}; - -void realloc_nodes(struct osmdata_t *osmdata); -void realloc_members(struct osmdata_t *osmdata); -void resetMembers(struct osmdata_t *osmdata); -void printStatus(struct osmdata_t *osmdata); -int node_wanted(struct osmdata_t *osmdata, double lat, double lon); - -/* exit_nicely - called to cleanup after fatal error */ -void exit_nicely(void); - -#endif diff -Nru osm2pgsql-0.82.0/osmtypes.hpp osm2pgsql-0.88.1/osmtypes.hpp --- osm2pgsql-0.82.0/osmtypes.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/osmtypes.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,123 @@ +/* Data types to hold OSM node, segment, way data */ + +#ifndef OSMTYPES_H +#define OSMTYPES_H + +// when __cplusplus is defined, we need to define this macro as well +// to get the print format specifiers in the inttypes.h header. +#define __STDC_FORMAT_MACROS +#include +#include "config.h" + +#include +#include +#include + +typedef int64_t osmid_t; +#define strtoosmid strtoll +#define PRIdOSMID PRId64 +#define POSTGRES_OSMID_TYPE "int8" + +enum OsmType { OSMTYPE_WAY, OSMTYPE_NODE, OSMTYPE_RELATION }; + +struct osmNode { + double lon; + double lat; + + osmNode() : lon(NAN), lat(NAN) {} + + osmNode(double x, double y) : lon(x), lat(y) {} +}; + +typedef std::vector nodelist_t; +typedef std::vector multinodelist_t; + +struct member { + OsmType type; + osmid_t id; + std::string role; + + member(OsmType t, osmid_t i, const std::string &r) : type(t), id(i), role(r) {} +}; + +typedef std::vector memberlist_t; + +struct tag { + std::string key; + std::string value; + + tag(const std::string &k, const std::string &v) : key(k), value(v) {} +}; + + +class taglist_t : public std::vector { + + typedef std::vector base_t; + +public: + const tag *find(const std::string &key) const { return _find(key); } + + tag *find(const std::string &key) { return const_cast(_find(key)); } + + size_t indexof(const std::string &key) const + { + for (size_t i = 0; i < size(); ++i) + if (at(i).key == key) + return i; + + return -1; + } + + const std::string *get(const std::string &key) const + { + for (base_t::const_iterator it = begin() ; it != end(); ++it) + if (it->key == key) + return &(it->value); + + return 0; + } + + bool get_bool(const std::string &key, bool defval) const + { + for (base_t::const_iterator it = begin() ; it != end(); ++it) + if (it->key == key) { + if (!defval && + (it->value == "yes" || it->value == "true" || it->value == "1")) + return true; + if (defval && + (it->value == "no" || it->value == "false" || it->value == "0")) + return false; + return defval; + } + + return defval; + } + + void push_dedupe(const tag& t) + { + if (find(t.key) == 0) + push_back(t); + } + + bool contains(const std::string &key) const { return _find(key) != 0; } + +private: + const tag *_find(const std::string &key) const + { + for (base_t::const_iterator it = begin() ; it != end(); ++it) + if (it->key == key) + return &(*it); + + return 0; + } + + +}; + +typedef std::vector multitaglist_t; + +typedef std::vector idlist_t; + +typedef std::vector rolelist_t; + +#endif diff -Nru osm2pgsql-0.82.0/output.cpp osm2pgsql-0.88.1/output.cpp --- osm2pgsql-0.82.0/output.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,161 @@ +#include "output.hpp" +#include "output-pgsql.hpp" +#include "output-gazetteer.hpp" +#include "output-null.hpp" +#include "output-multi.hpp" +#include "taginfo_impl.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pt = boost::property_tree; + +namespace { + +template +void override_if(T &t, const std::string &key, const pt::ptree &conf) { + boost::optional opt = conf.get_optional(key); + if (opt) { + t = *opt; + } +} + +boost::shared_ptr parse_multi_single(const pt::ptree &conf, + const middle_query_t *mid, + const options_t &options) { + options_t new_opts = options; + + std::string name = conf.get("name"); + std::string proc_type = conf.get("type"); + + new_opts.tag_transform_script = conf.get_optional("tagtransform"); + + new_opts.tag_transform_node_func = conf.get_optional("tagtransform-node-function"); + new_opts.tag_transform_way_func = conf.get_optional("tagtransform-way-function"); + new_opts.tag_transform_rel_func = conf.get_optional("tagtransform-relation-function"); + new_opts.tag_transform_rel_mem_func = conf.get_optional("tagtransform-relation-member-function"); + + new_opts.tblsmain_index = conf.get_optional("tablespace-index"); + new_opts.tblsmain_data = conf.get_optional("tablespace-data"); + override_if(new_opts.hstore_mode, "enable-hstore", conf); + override_if(new_opts.enable_hstore_index, "enable-hstore-index", conf); + override_if(new_opts.enable_multi, "enable-multi", conf); + override_if(new_opts.hstore_match_only, "hstore-match-only", conf); + + hstores_t hstore_columns; + boost::optional hstores = conf.get_child_optional("hstores"); + if (hstores) { + BOOST_FOREACH(const pt::ptree::value_type &val, *hstores) { + hstore_columns.push_back(val.second.get_value()); + } + } + new_opts.hstore_columns = hstore_columns; + + boost::shared_ptr processor = + geometry_processor::create(proc_type, &new_opts); + + // TODO: we're faking this up, but there has to be a better way? + OsmType osm_type = ((processor->interests() & geometry_processor::interest_node) > 0) + ? OSMTYPE_NODE : OSMTYPE_WAY; + + export_list columns; + const pt::ptree &tags = conf.get_child("tags"); + BOOST_FOREACH(const pt::ptree::value_type &val, tags) { + const pt::ptree &tag = val.second; + taginfo info; + info.name = tag.get("name"); + info.type = tag.get("type"); + std::string flags = tag.get_optional("flags").get_value_or(std::string()); + // TODO: we fake the line number here - any way to get the right one + // from the JSON parser? + info.flags = parse_tag_flags(flags.c_str(), -1); + // TODO: shouldn't need to specify a type here? + columns.add(osm_type, info); + } + + return boost::make_shared(name, processor, columns, mid, new_opts); +} + +std::vector > parse_multi_config(const middle_query_t *mid, const options_t &options) { + std::vector > outputs; + + if (!options.style.empty()) { + const std::string file_name(options.style); + + try { + pt::ptree conf; + pt::read_json(file_name, conf); + + BOOST_FOREACH(const pt::ptree::value_type &val, conf) { + outputs.push_back(parse_multi_single(val.second, mid, options)); + } + + } catch (const std::exception &e) { + throw std::runtime_error((boost::format("Unable to parse multi config file `%1%': %2%") + % file_name % e.what()).str()); + } + } else { + throw std::runtime_error("Style file is required for `multi' backend, but was not specified."); + } + + return outputs; +} + +} // anonymous namespace + +std::vector > output_t::create_outputs(const middle_query_t *mid, const options_t &options) { + std::vector > outputs; + + if (options.output_backend == "pgsql") { + outputs.push_back(boost::make_shared(mid, options)); + + } else if (options.output_backend == "gazetteer") { + outputs.push_back(boost::make_shared(mid, options)); + + } else if (options.output_backend == "null") { + outputs.push_back(boost::make_shared(mid, options)); + + } else if (options.output_backend == "multi") { + outputs = parse_multi_config(mid, options); + + } else { + throw std::runtime_error((boost::format("Output backend `%1%' not recognised. Should be one of [pgsql, gazetteer, null, multi].\n") % options.output_backend).str()); + } + + return outputs; +} + +output_t::output_t(const middle_query_t *mid_, const options_t &options_): m_mid(mid_), m_options(options_) { + +} + +output_t::~output_t() { + +} + +size_t output_t::pending_count() const{ + return 0; +} + +const options_t *output_t::get_options()const { + return &m_options; +} + +void output_t::merge_pending_relations(boost::shared_ptr other) { +} +void output_t::merge_expire_trees(boost::shared_ptr other) { +} + +boost::shared_ptr output_t::get_pending_relations() { + return boost::shared_ptr(); +} +boost::shared_ptr output_t::get_expire_tree() { + return boost::shared_ptr(); +} diff -Nru osm2pgsql-0.82.0/output-gazetteer.c osm2pgsql-0.88.1/output-gazetteer.c --- osm2pgsql-0.82.0/output-gazetteer.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/output-gazetteer.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,1358 +0,0 @@ -#include -#include -#include - -#include - -#include "osmtypes.h" -#include "middle.h" -#include "output.h" -#include "output-gazetteer.h" -#include "pgsql.h" -#include "reprojection.h" -#include "build_geometry.h" - -#define BUFFER_SIZE 4096 - -#define SRID (project_getprojinfo()->srs) - -#define CREATE_KEYVALUETYPE_TYPE \ - "CREATE TYPE keyvalue AS (" \ - " key TEXT," \ - " value TEXT" \ - ")" - -#define CREATE_WORDSCORE_TYPE \ - "CREATE TYPE wordscore AS (" \ - " word TEXT," \ - " score FLOAT" \ - ")" - -#define CREATE_PLACE_TABLE \ - "CREATE TABLE place (" \ - " osm_type CHAR(1) NOT NULL," \ - " osm_id " POSTGRES_OSMID_TYPE " NOT NULL," \ - " class TEXT NOT NULL," \ - " type TEXT NOT NULL," \ - " name HSTORE," \ - " admin_level INTEGER," \ - " housenumber TEXT," \ - " street TEXT," \ - " isin TEXT," \ - " postcode TEXT," \ - " country_code VARCHAR(2)," \ - " extratags HSTORE" \ - ") %s %s" - -#define ADMINLEVEL_NONE 100 - -#define CREATE_PLACE_ID_INDEX \ - "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id) %s %s" - -#define TAGINFO_NODE 0x1u -#define TAGINFO_WAY 0x2u -#define TAGINFO_AREA 0x4u - - -static const struct output_options *Options = NULL; -static PGconn *Connection = NULL; -static int CopyActive = 0; -static char Buffer[BUFFER_SIZE]; -static unsigned int BufferLen = 0; - -static PGconn *ConnectionDelete = NULL; - -static PGconn *ConnectionError = NULL; -static int CopyErrorActive = 0; -static char BufferError[BUFFER_SIZE]; -static unsigned int BufferErrorLen = 0; - -static FILE * hLog = NULL; - -static void require_slim_mode(void) -{ - if (!Options->slim) - { - fprintf(stderr, "Cannot apply diffs unless in slim mode\n"); - exit_nicely(); - } - - return; -} - -static void copy_data(const char *sql) -{ - unsigned int sqlLen = strlen(sql); - - /* Make sure we have an active copy */ - if (!CopyActive) - { - pgsql_exec(Connection, PGRES_COPY_IN, "COPY place FROM STDIN"); - CopyActive = 1; - } - - /* If the combination of old and new data is too big, flush old data */ - if (BufferLen + sqlLen > BUFFER_SIZE - 10) - { - pgsql_CopyData("place", Connection, Buffer); - BufferLen = 0; - } - - /* - * If new data by itself is too big, output it immediately, - * otherwise just add it to the buffer. - */ - if (sqlLen > BUFFER_SIZE - 10) - { - pgsql_CopyData("Place", Connection, sql); - sqlLen = 0; - } - else if (sqlLen > 0) - { - strcpy(Buffer + BufferLen, sql); - BufferLen += sqlLen; - sqlLen = 0; - } - - /* If we have completed a line, output it */ - if (BufferLen > 0 && Buffer[BufferLen-1] == '\n') - { - pgsql_CopyData("place", Connection, Buffer); - BufferLen = 0; - } - - return; -} - -static void stop_copy(void) -{ - PGresult *res; - - /* Do we have a copy active? */ - if (!CopyActive) return; - - /* Terminate the copy */ - if (PQputCopyEnd(Connection, NULL) != 1) - { - fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection)); - exit_nicely(); - } - - /* Check the result */ - res = PQgetResult(Connection); - if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - fprintf(stderr, "COPY_END for place failed: %s\n", PQerrorMessage(Connection)); - PQclear(res); - exit_nicely(); - } - - /* Discard the result */ - PQclear(res); - - /* We no longer have an active copy */ - CopyActive = 0; - - return; -} - -static void copy_error_data(const char *sql) -{ - unsigned int sqlLen = strlen(sql); - - if (hLog) fprintf(hLog, "%s", sql); - - /* Make sure we have an active copy */ - if (!CopyErrorActive) - { - pgsql_exec(ConnectionError, PGRES_COPY_IN, "COPY import_polygon_error (osm_type, osm_id, class, type, name, country_code, updated, errormessage, prevgeometry, newgeometry) FROM stdin;"); - CopyErrorActive = 1; - } - - /* If the combination of old and new data is too big, flush old data */ - if (BufferErrorLen + sqlLen > BUFFER_SIZE - 10) - { - pgsql_CopyData("import_polygon_error", ConnectionError, BufferError); - BufferErrorLen = 0; - } - - /* - * If new data by itself is too big, output it immediately, - * otherwise just add it to the buffer. - */ - if (sqlLen > BUFFER_SIZE - 10) - { - pgsql_CopyData("import_polygon_error", ConnectionError, sql); - sqlLen = 0; - } - else if (sqlLen > 0) - { - strcpy(BufferError + BufferErrorLen, sql); - BufferErrorLen += sqlLen; - sqlLen = 0; - } - - /* If we have completed a line, output it */ - if (BufferErrorLen > 0 && BufferError[BufferErrorLen-1] == '\n') - { - pgsql_CopyData("place", ConnectionError, BufferError); - BufferErrorLen = 0; - } - - return; -} - -static void stop_error_copy(void) -{ - PGresult *res; - - /* Do we have a copy active? */ - if (!CopyErrorActive) return; - - /* Terminate the copy */ - if (PQputCopyEnd(ConnectionError, NULL) != 1) - { - fprintf(stderr, "COPY_END for import_polygon_error failed: %s\n", PQerrorMessage(ConnectionError)); - exit_nicely(); - } - - /* Check the result */ - res = PQgetResult(ConnectionError); - if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - fprintf(stderr, "COPY_END for import_polygon_error failed: %s\n", PQerrorMessage(ConnectionError)); - PQclear(res); - exit_nicely(); - } - - /* Discard the result */ - PQclear(res); - - /* We no longer have an active copy */ - CopyErrorActive = 0; - - return; -} - -static int split_tags(struct keyval *tags, unsigned int flags, struct keyval *names, struct keyval *places, struct keyval *extratags, - int* admin_level, struct keyval ** housenumber, struct keyval ** street, char ** isin, struct keyval ** postcode, struct keyval ** countrycode) -{ - int placehouse = 0; - int placebuilding = 0; - int placeadmin = 0; - struct keyval *landuse; - struct keyval *place; - struct keyval *item; - struct keyval *conscriptionnumber; - struct keyval *streetnumber; - - *admin_level = ADMINLEVEL_NONE; - *housenumber = 0; - *street = 0; - *isin = 0; - int isinsize = 0; - *postcode = 0; - *countrycode = 0; - landuse = 0; - place = 0; - conscriptionnumber = 0; - streetnumber = 0; - - /* Initialise the result lists */ - initList(names); - initList(places); - initList(extratags); - - /* Loop over the tags */ - while ((item = popItem(tags)) != NULL) - { - - /* If this is a name tag, add it to the name list */ - if (strcmp(item->key, "ref") == 0 || - strcmp(item->key, "int_ref") == 0 || - strcmp(item->key, "nat_ref") == 0 || - strcmp(item->key, "reg_ref") == 0 || - strcmp(item->key, "loc_ref") == 0 || - strcmp(item->key, "old_ref") == 0 || - strcmp(item->key, "ncn_ref") == 0 || - strcmp(item->key, "rcn_ref") == 0 || - strcmp(item->key, "lcn_ref") == 0 || - strcmp(item->key, "iata") == 0 || - strcmp(item->key, "icao") == 0 || - strcmp(item->key, "pcode:1") == 0 || - strcmp(item->key, "pcode:2") == 0 || - strcmp(item->key, "pcode:3") == 0 || - strcmp(item->key, "un:pcode:1") == 0 || - strcmp(item->key, "un:pcode:2") == 0 || - strcmp(item->key, "un:pcode:3") == 0 || - strcmp(item->key, "name") == 0 || - (strncmp(item->key, "name:", 5) == 0) || - strcmp(item->key, "int_name") == 0 || - (strncmp(item->key, "int_name:", 9) == 0) || - strcmp(item->key, "nat_name") == 0 || - (strncmp(item->key, "nat_name:", 9) == 0) || - strcmp(item->key, "reg_name") == 0 || - (strncmp(item->key, "reg_name:", 9) == 0) || - strcmp(item->key, "loc_name") == 0 || - (strncmp(item->key, "loc_name:", 9) == 0) || - strcmp(item->key, "old_name") == 0 || - (strncmp(item->key, "old_name:", 9) == 0) || - strcmp(item->key, "alt_name") == 0 || - (strncmp(item->key, "alt_name:", 9) == 0) || - strcmp(item->key, "official_name") == 0 || - (strncmp(item->key, "official_name:", 14) == 0) || - strcmp(item->key, "commonname") == 0 || - (strncmp(item->key, "commonname:", 11) == 0) || - strcmp(item->key, "common_name") == 0 || - (strncmp(item->key, "common_name:", 12) == 0) || - strcmp(item->key, "place_name") == 0 || - (strncmp(item->key, "place_name:", 11) == 0) || - strcmp(item->key, "short_name") == 0 || - (strncmp(item->key, "short_name:", 11) == 0) || - strcmp(item->key, "operator") == 0) /* operator is a bit of an oddity */ - { - if (strcmp(item->key, "name:prefix") == 0) - { - pushItem(extratags, item); - } - else - { - pushItem(names, item); - } - } - else if (strcmp(item->key, "aerialway") == 0 || - strcmp(item->key, "aeroway") == 0 || - strcmp(item->key, "amenity") == 0 || - strcmp(item->key, "boundary") == 0 || - strcmp(item->key, "bridge") == 0 || - strcmp(item->key, "craft") == 0 || - strcmp(item->key, "emergency") == 0 || - strcmp(item->key, "highway") == 0 || - strcmp(item->key, "historic") == 0 || - strcmp(item->key, "leisure") == 0 || - strcmp(item->key, "military") == 0 || - strcmp(item->key, "natural") == 0 || - strcmp(item->key, "office") == 0 || - strcmp(item->key, "railway") == 0 || - strcmp(item->key, "shop") == 0 || - strcmp(item->key, "tourism") == 0 || - strcmp(item->key, "tunnel") == 0 || - strcmp(item->key, "waterway") == 0 ) - { - if (strcmp(item->value, "no")) - { - pushItem(places, item); - if (strcmp(item->key, "boundary") == 0 && strcmp(item->value, "administrative") == 0) - { - placeadmin = 1; - } - } - else - { - freeItem(item); - } - } - else if (strcmp(item->key, "place") == 0) - { - place = item; - } - else if (strcmp(item->key, "addr:housename") == 0) - { - pushItem(names, item); - placehouse = 1; - } - else if (strcmp(item->key, "landuse") == 0) - { - if (strcmp(item->value, "cemetery") == 0) - pushItem(places, item); - else - landuse = item; - } - else if (strcmp(item->key, "postal_code") == 0 || - strcmp(item->key, "post_code") == 0 || - strcmp(item->key, "postcode") == 0 || - strcmp(item->key, "addr:postcode") == 0 || - strcmp(item->key, "tiger:zip_left") == 0 || - strcmp(item->key, "tiger:zip_right") == 0) - { - if (*postcode) - freeItem(item); - else - *postcode = item; - } - else if (strcmp(item->key, "addr:street") == 0) - { - *street = item; - } - else if ((strcmp(item->key, "country_code_iso3166_1_alpha_2") == 0 || - strcmp(item->key, "country_code_iso3166_1") == 0 || - strcmp(item->key, "country_code_iso3166") == 0 || - strcmp(item->key, "country_code") == 0 || - strcmp(item->key, "iso3166-1:alpha2") == 0 || - strcmp(item->key, "iso3166-1") == 0 || - strcmp(item->key, "ISO3166-1") == 0 || - strcmp(item->key, "iso3166") == 0 || - strcmp(item->key, "is_in:country_code") == 0 || - strcmp(item->key, "addr:country") == 0 || - strcmp(item->key, "addr:country_code") == 0) - && strlen(item->value) == 2) - { - *countrycode = item; - } - else if (strcmp(item->key, "addr:housenumber") == 0) - { - /* house number can be far more complex than just a single house number - leave for postgresql to deal with */ - if (*housenumber) - freeItem(item); - else { - *housenumber = item; - placehouse = 1; - } - } - else if (strcmp(item->key, "addr:conscriptionnumber") == 0) - { - if (conscriptionnumber) - freeItem(item); - else { - conscriptionnumber = item; - placehouse = 1; - } - } - else if (strcmp(item->key, "addr:streetnumber") == 0) - { - if (streetnumber) - freeItem(item); - else { - streetnumber = item; - placehouse = 1; - } - } - else if (strcmp(item->key, "addr:interpolation") == 0) - { - /* house number can be far more complex than just a single house number - leave for postgresql to deal with */ - if (*housenumber) { - freeItem(item); - } else { - *housenumber = item; - addItem(places, "place", "houses", 1); - } - } - else if (strcmp(item->key, "is_in") == 0 || - (strncmp(item->key, "is_in:", 5) == 0) || - strcmp(item->key, "addr:country")== 0 || - strcmp(item->key, "addr:county")== 0 || - strcmp(item->key, "tiger:county")== 0 || - strcmp(item->key, "addr:city") == 0 || - strcmp(item->key, "addr:state_code") == 0 || - strcmp(item->key, "addr:state") == 0) - { - *isin = realloc(*isin, isinsize + 2 + strlen(item->value)); - *(*isin+isinsize) = ','; - strcpy(*isin+1+isinsize, item->value); - isinsize += 1 + strlen(item->value); - freeItem(item); - } - else if (strcmp(item->key, "admin_level") == 0) - { - *admin_level = atoi(item->value); - freeItem(item); - } - else if (strcmp(item->key, "tracktype") == 0 || - strcmp(item->key, "traffic_calming") == 0 || - strcmp(item->key, "service") == 0 || - strcmp(item->key, "cuisine") == 0 || - strcmp(item->key, "capital") == 0 || - strcmp(item->key, "dispensing") == 0 || - strcmp(item->key, "religion") == 0 || - strcmp(item->key, "denomination") == 0 || - strcmp(item->key, "sport") == 0 || - strcmp(item->key, "internet_access") == 0 || - strcmp(item->key, "lanes") == 0 || - strcmp(item->key, "surface") == 0 || - strcmp(item->key, "smoothness") == 0 || - strcmp(item->key, "width") == 0 || - strcmp(item->key, "est_width") == 0 || - strcmp(item->key, "incline") == 0 || - strcmp(item->key, "opening_hours") == 0 || - strcmp(item->key, "food_hours") == 0 || - strcmp(item->key, "collection_times") == 0 || - strcmp(item->key, "service_times") == 0 || - strcmp(item->key, "smoking_hours") == 0 || - strcmp(item->key, "disused") == 0 || - strcmp(item->key, "wheelchair") == 0 || - strcmp(item->key, "sac_scale") == 0 || - strcmp(item->key, "trail_visibility") == 0 || - strcmp(item->key, "mtb:scale") == 0 || - strcmp(item->key, "mtb:description") == 0 || - strcmp(item->key, "wood") == 0 || - strcmp(item->key, "drive_thru") == 0 || - strcmp(item->key, "drive_in") == 0 || - strcmp(item->key, "access") == 0 || - strcmp(item->key, "vehicle") == 0 || - strcmp(item->key, "bicyle") == 0 || - strcmp(item->key, "foot") == 0 || - strcmp(item->key, "goods") == 0 || - strcmp(item->key, "hgv") == 0 || - strcmp(item->key, "motor_vehicle") == 0 || - strcmp(item->key, "motor_car") == 0 || - (strncmp(item->key, "access:", 7) == 0) || - (strncmp(item->key, "contact:", 8) == 0) || - (strncmp(item->key, "drink:", 6) == 0) || - strcmp(item->key, "oneway") == 0 || - strcmp(item->key, "date_on") == 0 || - strcmp(item->key, "date_off") == 0 || - strcmp(item->key, "day_on") == 0 || - strcmp(item->key, "day_off") == 0 || - strcmp(item->key, "hour_on") == 0 || - strcmp(item->key, "hour_off") == 0 || - strcmp(item->key, "maxweight") == 0 || - strcmp(item->key, "maxheight") == 0 || - strcmp(item->key, "maxspeed") == 0 || - strcmp(item->key, "disused") == 0 || - strcmp(item->key, "toll") == 0 || - strcmp(item->key, "charge") == 0 || - strcmp(item->key, "population") == 0 || - strcmp(item->key, "description") == 0 || - strcmp(item->key, "image") == 0 || - strcmp(item->key, "attribution") == 0 || - strcmp(item->key, "fax") == 0 || - strcmp(item->key, "email") == 0 || - strcmp(item->key, "url") == 0 || - strcmp(item->key, "website") == 0 || - strcmp(item->key, "phone") == 0 || - strcmp(item->key, "tel") == 0 || - strcmp(item->key, "real_ale") == 0 || - strcmp(item->key, "smoking") == 0 || - strcmp(item->key, "food") == 0 || - strcmp(item->key, "camera") == 0 || - strcmp(item->key, "brewery") == 0 || - strcmp(item->key, "locality") == 0 || - strcmp(item->key, "wikipedia") == 0 || - (strncmp(item->key, "wikipedia:", 10) == 0) - ) - { - pushItem(extratags, item); - } - else if (strcmp(item->key, "building") == 0) - { - placebuilding = 1; - freeItem(item); - } - else if (strcmp(item->key, "mountain_pass") == 0) - { - pushItem(places, item); - } - else - { - freeItem(item); - } - } - - /* Handle Czech/Slovak addresses: - - if we have just a conscription number or a street number, - just use the one we have as a house number - - if we have both of them, concatenate them so users may search - by any of them - */ - if (conscriptionnumber || streetnumber) - { - if (*housenumber) - { - freeItem(*housenumber); - } - if (!conscriptionnumber) - { - addItem(tags, "addr:housenumber", streetnumber->value, 0); - freeItem(streetnumber); - *housenumber = popItem(tags); - } - if (!streetnumber) - { - addItem(tags, "addr:housenumber", conscriptionnumber->value, 10); - freeItem(conscriptionnumber); - *housenumber = popItem(tags); - } - if (conscriptionnumber && streetnumber) - { - char * completenumber = strdup(conscriptionnumber->value); - size_t completenumberlength = strlen(completenumber); - completenumber = realloc(completenumber, completenumberlength + 2 + strlen(streetnumber->value)); - *(completenumber + completenumberlength) = '/'; - strcpy(completenumber + completenumberlength + 1, streetnumber->value); - freeItem(conscriptionnumber); - freeItem(streetnumber); - addItem(tags, "addr:housenumber", completenumber, 0); - *housenumber = popItem(tags); - free(completenumber); - } - } - - if (place) - { - if (placeadmin) - { - pushItem(extratags, place); - } - else - { - pushItem(places, place); - } - } - - if (placehouse && !listHasData(places)) - { - addItem(places, "place", "house", 1); - } - - /* Fallback place types - only used if we didn't create something more specific already */ - if (placebuilding && !listHasData(places) && (listHasData(names) || *housenumber || *postcode)) - { - addItem(places, "building", "yes", 1); - } - - if (landuse) - { - if (!listHasData(places)) - { - pushItem(places, landuse); - } - else - { - freeItem(item); - } - } - - if (*postcode && !listHasData(places)) - { - addItem(places, "place", "postcode", 1); - } - - /* Try to convert everything to an area */ - return 1; -} - -void escape_array_record(char *out, int len, const char *in) -{ - int count = 0; - const char *old_in = in, *old_out = out; - - if (!len) - return; - - while(*in && count < len-3) { - switch(*in) { - case '\\': *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; *out++ = '\\'; count+= 8; break; - case '\n': - case '\r': - case '\t': - case '"': - /* This is a bit naughty - we know that nominatim ignored these characters so just drop them now for simplicity */ - *out++ = ' '; count++; break; - default: *out++ = *in; count++; break; - } - in++; - } - *out = '\0'; - - if (*in) - fprintf(stderr, "%s truncated at %d chars: %s\n%s\n", __FUNCTION__, count, old_in, old_out); -} - -static void delete_unused_classes(char osm_type, osmid_t osm_id, struct keyval *places) { - int i,sz, slen; - PGresult *res; - char tmp[16]; - char tmp2[2]; - char *cls, *clslist = 0; - char const *paramValues[2]; - - tmp2[0] = osm_type; tmp2[1] = '\0'; - paramValues[0] = tmp2; - snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, osm_id); - paramValues[1] = tmp; - res = pgsql_execPrepared(ConnectionDelete, "get_classes", 2, paramValues, PGRES_TUPLES_OK); - - sz = PQntuples(res); - if (sz > 0 && !places) { - PQclear(res); - /* uncondtional delete of all places */ - stop_copy(); - pgsql_exec(Connection, PGRES_COMMAND_OK, "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %" PRIdOSMID, osm_type, osm_id); - } else { - for (i = 0; i < sz; i++) { - cls = PQgetvalue(res, i, 0); - if (!getItem(places, cls)) { - if (!clslist) { - clslist = malloc(strlen(cls)+3); - sprintf(clslist, "'%s'", cls); - } else { - slen = strlen(clslist); - clslist = realloc(clslist, slen + 4 + strlen(cls)); - sprintf(&(clslist[slen]), ",'%s'", cls); - } - } - } - - PQclear(res); - - if (clslist) { - /* Stop any active copy */ - stop_copy(); - - /* Delete all places for this object */ - pgsql_exec(Connection, PGRES_COMMAND_OK, "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %" - PRIdOSMID " and class = any(ARRAY[%s])", osm_type, osm_id, clslist); - free(clslist); - } - } -} - -static void add_place(char osm_type, osmid_t osm_id, const char *class, const char *type, struct keyval *names, struct keyval *extratags, - int adminlevel, struct keyval *housenumber, struct keyval *street, const char *isin, struct keyval *postcode, struct keyval *countrycode, const char *wkt) -{ - int first; - struct keyval *name; - char sql[2048]; - - /* Output a copy line for this place */ - sprintf(sql, "%c\t%" PRIdOSMID "\t", osm_type, osm_id); - copy_data(sql); - - escape(sql, sizeof(sql), class); - copy_data(sql); - copy_data("\t"); - - escape(sql, sizeof(sql), type); - copy_data(sql); - copy_data("\t"); - - /* start name array */ - if (listHasData(names)) - { - first = 1; - for (name = firstItem(names); name; name = nextItem(names, name)) - { - if (first) first = 0; - else copy_data(", "); - - copy_data("\""); - - escape_array_record(sql, sizeof(sql), name->key); - copy_data(sql); - - copy_data("\"=>\""); - - escape_array_record(sql, sizeof(sql), name->value); - copy_data(sql); - - copy_data("\""); - } - copy_data("\t"); - } - else - { - copy_data("\\N\t"); - } - - sprintf(sql, "%d\t", adminlevel); - copy_data(sql); - - if (housenumber) - { - escape(sql, sizeof(sql), housenumber->value); - copy_data(sql); - copy_data("\t"); - } - else - { - copy_data("\\N\t"); - } - - if (street) - { - escape(sql, sizeof(sql), street->value); - copy_data(sql); - copy_data("\t"); - } - else - { - copy_data("\\N\t"); - } - - if (isin) - { - /* Skip the leading ',' from the contactination */ - escape(sql, sizeof(sql), isin+1); - copy_data(sql); - copy_data("\t"); - } - else - { - copy_data("\\N\t"); - } - - if (postcode) - { - escape(sql, sizeof(sql), postcode->value); - copy_data(sql); - copy_data("\t"); - } - else - { - copy_data("\\N\t"); - } - - if (countrycode) - { - escape(sql, sizeof(sql), countrycode->value); - copy_data(sql); - copy_data("\t"); - } - else - { - copy_data("\\N\t"); - } - - /* extra tags array */ - if (listHasData(extratags)) - { - first = 1; - for (name = firstItem(extratags); name; name = nextItem(extratags, name)) - { - if (first) first = 0; - else copy_data(", "); - - copy_data("\""); - - escape_array_record(sql, sizeof(sql), name->key); - copy_data(sql); - - copy_data("\"=>\""); - - escape_array_record(sql, sizeof(sql), name->value); - copy_data(sql); - - copy_data("\""); - } - copy_data("\t"); - } - else - { - copy_data("\\N\t"); - } - - sprintf(sql, "SRID=%d;", SRID); - copy_data(sql); - copy_data(wkt); - - copy_data("\n"); - - - return; -} - -static void add_polygon_error(char osm_type, osmid_t osm_id, const char *class, const char *type, - struct keyval *names, const char *countrycode, const char *wkt) -{ - int first; - struct keyval *name; - char sql[2048]; - - /* Output a copy line for this place */ - sprintf(sql, "%c\t%" PRIdOSMID "\t", osm_type, osm_id); - copy_error_data(sql); - - escape(sql, sizeof(sql), class); - copy_error_data(sql); - copy_error_data("\t"); - - escape(sql, sizeof(sql), type); - copy_error_data(sql); - copy_error_data("\t"); - - /* start name array */ - if (listHasData(names)) - { - first = 1; - for (name = firstItem(names); name; name = nextItem(names, name)) - { - if (first) first = 0; - else copy_error_data(", "); - - copy_error_data("\""); - - escape_array_record(sql, sizeof(sql), name->key); - copy_error_data(sql); - - copy_error_data("\"=>\""); - - escape_array_record(sql, sizeof(sql), name->value); - copy_error_data(sql); - - copy_error_data("\""); - } - copy_error_data("\t"); - } - else - { - copy_error_data("\\N\t"); - } - - if (countrycode) - { - escape(sql, sizeof(sql), countrycode); - copy_error_data(sql); - copy_error_data("\t"); - } - else - { - copy_error_data("\\N\t"); - } - - copy_error_data("now\tNot a polygon\t\\N\t"); - - sprintf(sql, "SRID=%d;", SRID); - copy_error_data(sql); - copy_error_data(wkt); - - copy_error_data("\n"); - - - return; -} - - -static void delete_place(char osm_type, osmid_t osm_id) -{ - /* Stop any active copy */ - stop_copy(); - - /* Delete all places for this object */ - pgsql_exec(Connection, PGRES_COMMAND_OK, "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %" PRIdOSMID, osm_type, osm_id); - - return; -} - -static int gazetteer_out_start(const struct output_options *options) -{ - /* Save option handle */ - Options = options; - - /* Connection to the database */ - Connection = PQconnectdb(options->conninfo); - - /* Check to see that the backend connection was successfully made */ - if (PQstatus(Connection) != CONNECTION_OK) - { - fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(Connection)); - exit_nicely(); - } - - /* Start a transaction */ - pgsql_exec(Connection, PGRES_COMMAND_OK, "BEGIN"); - - /* (Re)create the table unless we are appending */ - if (!Options->append) - { - /* Drop any existing table */ - pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS place"); - pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists keyvalue cascade"); - pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists wordscore cascade"); - pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists stringlanguagetype cascade"); - pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TYPE if exists keyvaluetype cascade"); - pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP FUNCTION IF EXISTS get_connected_ways(integer[])"); - - /* Create types and functions */ - pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_KEYVALUETYPE_TYPE); - pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_WORDSCORE_TYPE); - - /* Create the new table */ - if (Options->tblsmain_data) - { - pgsql_exec(Connection, PGRES_COMMAND_OK, - CREATE_PLACE_TABLE, "TABLESPACE", Options->tblsmain_data); - } - else - { - pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_TABLE, "", ""); - } - if (Options->tblsmain_index) - { - pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_ID_INDEX, "TABLESPACE", Options->tblsmain_index); - } - else - { - pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_ID_INDEX, "", ""); - } - - pgsql_exec(Connection, PGRES_TUPLES_OK, "SELECT AddGeometryColumn('place', 'geometry', %d, 'GEOMETRY', 2)", SRID); - pgsql_exec(Connection, PGRES_COMMAND_OK, "ALTER TABLE place ALTER COLUMN geometry SET NOT NULL"); - } else { - ConnectionDelete = PQconnectdb(options->conninfo); - if (PQstatus(ConnectionDelete) != CONNECTION_OK) - { - fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(ConnectionDelete)); - exit_nicely(); - } - - pgsql_exec(ConnectionDelete, PGRES_COMMAND_OK, "PREPARE get_classes (CHAR(1), " POSTGRES_OSMID_TYPE ") AS SELECT class FROM place WHERE osm_type = $1 and osm_id = $2"); - } - - /* Setup middle layer */ - options->mid->start(options); - - hLog = fopen("log", "w"); - - return 0; -} - -static void gazetteer_out_stop(void) -{ - /* Process any remaining ways and relations */ - - /* No longer need to access middle layer */ - Options->mid->commit(); - Options->mid->stop(); - - /* Stop any active copy */ - stop_copy(); - if (hLog) fclose(hLog); - - /* Commit transaction */ - pgsql_exec(Connection, PGRES_COMMAND_OK, "COMMIT"); - - - PQfinish(Connection); - if (ConnectionDelete) - PQfinish(ConnectionDelete); - if (ConnectionError) - PQfinish(ConnectionError); - - return; -} - -static void gazetteer_out_cleanup(void) -{ - return; -} - -static int gazetteer_process_node(osmid_t id, double lat, double lon, struct keyval *tags, int delete_old) -{ - struct keyval names; - struct keyval places; - struct keyval extratags; - struct keyval *place; - int adminlevel; - struct keyval * housenumber; - struct keyval * street; - char * isin; - struct keyval * postcode; - struct keyval * countrycode; - char wkt[128]; - - - /* Split the tags */ - split_tags(tags, TAGINFO_NODE, &names, &places, &extratags, &adminlevel, &housenumber, &street, &isin, &postcode, &countrycode); - - /* Feed this node to the middle layer */ - Options->mid->nodes_set(id, lat, lon, tags); - - if (delete_old) - delete_unused_classes('N', id, &places); - - /* Are we interested in this item? */ - if (listHasData(&places)) - { - sprintf(wkt, "POINT(%.15g %.15g)", lon, lat); - for (place = firstItem(&places); place; place = nextItem(&places, place)) - { - add_place('N', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, isin, postcode, countrycode, wkt); - } - } - - if (housenumber) freeItem(housenumber); - if (street) freeItem(street); - if (isin) free(isin); - if (postcode) freeItem(postcode); - if (countrycode) freeItem(countrycode); - - /* Free tag lists */ - resetList(&names); - resetList(&places); - resetList(&extratags); - - return 0; -} - -static int gazetteer_add_node(osmid_t id, double lat, double lon, struct keyval *tags) -{ - return gazetteer_process_node(id, lat, lon, tags, 0); -} - -static int gazetteer_process_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags, int delete_old) -{ - struct keyval names; - struct keyval places; - struct keyval extratags; - struct keyval *place; - int adminlevel; - struct keyval * housenumber; - struct keyval * street; - char * isin; - struct keyval * postcode; - struct keyval * countrycode; - int area; - - - /* Split the tags */ - area = split_tags(tags, TAGINFO_WAY, &names, &places, &extratags, &adminlevel, &housenumber, &street, &isin, &postcode, &countrycode); - - /* Feed this way to the middle layer */ - Options->mid->ways_set(id, ndv, ndc, tags, 0); - - if (delete_old) - delete_unused_classes('W', id, &places); - - /* Are we interested in this item? */ - if (listHasData(&places)) - { - struct osmNode *nodev; - int nodec; - char *wkt; - - /* Fetch the node details */ - nodev = malloc(ndc * sizeof(struct osmNode)); - nodec = Options->mid->nodes_get_list(nodev, ndv, ndc); - - /* Get the geometry of the object */ - if ((wkt = get_wkt_simple(nodev, nodec, area)) != NULL && strlen(wkt) > 0) - { - for (place = firstItem(&places); place; place = nextItem(&places, place)) - { - add_place('W', id, place->key, place->value, &names, &extratags, adminlevel, housenumber, street, isin, postcode, countrycode, wkt); - } - } - - /* Free the geometry */ - free(wkt); - - /* Free the nodes */ - free(nodev); - } - - if (housenumber) freeItem(housenumber); - if (street) freeItem(street); - if (isin) free(isin); - if (postcode) freeItem(postcode); - if (countrycode) freeItem(countrycode); - - /* Free tag lists */ - resetList(&names); - resetList(&places); - resetList(&extratags); - - return 0; -} - -static int gazetteer_add_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags) -{ - return gazetteer_process_way(id, ndv, ndc, tags, 0); -} - -static int gazetteer_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int delete_old) -{ - struct keyval names; - struct keyval places; - struct keyval extratags; - struct keyval *place; - int adminlevel; - struct keyval * housenumber; - struct keyval * street; - char * isin; - struct keyval * postcode; - struct keyval * countrycode; - int wkt_size; - const char *type; - - type = getItem(tags, "type"); - if (!type) { - if (delete_old) delete_unused_classes('R', id, 0); - return 0; - } - - if (!strcmp(type, "associatedStreet") || !strcmp(type, "relatedStreet")) - { - Options->mid->relations_set(id, members, member_count, tags); - if (delete_old) delete_unused_classes('R', id, 0); - return 0; - } - - if (strcmp(type, "boundary") && strcmp(type, "multipolygon")) { - if (delete_old) delete_unused_classes('R', id, 0); - return 0; - } - - Options->mid->relations_set(id, members, member_count, tags); - - /* Split the tags */ - split_tags(tags, TAGINFO_AREA, &names, &places, &extratags, &adminlevel, &housenumber, &street, &isin, &postcode, &countrycode); - - if (delete_old) - delete_unused_classes('R', id, &places); - - if (listHasData(&places)) - { - /* get the boundary path (ways) */ - int i, count; - int *xcount = malloc( (member_count+1) * sizeof(int) ); - struct keyval *xtags = malloc( (member_count+1) * sizeof(struct keyval) ); - struct osmNode **xnodes = malloc( (member_count+1) * sizeof(struct osmNode*) ); - osmid_t *xid; - osmid_t *xid2 = malloc( (member_count+1) * sizeof(osmid_t) ); - - count = 0; - for (i=0; imid->ways_get_list(xid2, count, &xid, xtags, xnodes, xcount); - - xnodes[count] = NULL; - xcount[count] = 0; - - wkt_size = build_geometry(id, xnodes, xcount, 1, 1, 1000000); - for (i=0;ikey, place->value, &names, &extratags, adminlevel, housenumber, street, isin, postcode, countrycode, wkt); - } - } - else - { - /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */ - } - free(wkt); - } - clear_wkts(); - - for( i=0; imid->nodes_delete(id); - - return 0; -} - -static int gazetteer_delete_way(osmid_t id) -{ - /* Make sure we are in slim mode */ - require_slim_mode(); - - /* Delete all references to this way */ - delete_place('W', id); - - /* Feed this delete to the middle layer */ - Options->mid->ways_delete(id); - - return 0; -} - -static int gazetteer_delete_relation(osmid_t id) -{ - /* Make sure we are in slim mode */ - require_slim_mode(); - - /* Delete all references to this relation */ - delete_place('R', id); - - /* Feed this delete to the middle layer */ - Options->mid->relations_delete(id); - - return 0; -} - -static int gazetteer_modify_node(osmid_t id, double lat, double lon, struct keyval *tags) -{ - require_slim_mode(); - Options->mid->nodes_delete(id); - return gazetteer_process_node(id, lat, lon, tags, 1); -} - -static int gazetteer_modify_way(osmid_t id, osmid_t *ndv, int ndc, struct keyval *tags) -{ - require_slim_mode(); - Options->mid->ways_delete(id); - return gazetteer_process_way(id, ndv, ndc, tags, 1); -} - -static int gazetteer_modify_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags) -{ - require_slim_mode(); - Options->mid->relations_delete(id); - return gazetteer_process_relation(id, members, member_count, tags, 1); -} - -struct output_t out_gazetteer = { - .start = gazetteer_out_start, - .stop = gazetteer_out_stop, - .cleanup = gazetteer_out_cleanup, - - .node_add = gazetteer_add_node, - .way_add = gazetteer_add_way, - .relation_add = gazetteer_add_relation, - - .node_modify = gazetteer_modify_node, - .way_modify = gazetteer_modify_way, - .relation_modify = gazetteer_modify_relation, - - .node_delete = gazetteer_delete_node, - .way_delete = gazetteer_delete_way, - .relation_delete = gazetteer_delete_relation -}; diff -Nru osm2pgsql-0.82.0/output-gazetteer.cpp osm2pgsql-0.88.1/output-gazetteer.cpp --- osm2pgsql-0.82.0/output-gazetteer.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-gazetteer.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,769 @@ +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "middle.hpp" +#include "pgsql.hpp" +#include "reprojection.hpp" +#include "output-gazetteer.hpp" +#include "options.hpp" +#include "util.hpp" + +#include + +#define SRID (reproj->project_getprojinfo()->srs) + +#define CREATE_PLACE_TABLE \ + "CREATE TABLE place (" \ + " osm_type CHAR(1) NOT NULL," \ + " osm_id " POSTGRES_OSMID_TYPE " NOT NULL," \ + " class TEXT NOT NULL," \ + " type TEXT NOT NULL," \ + " name HSTORE," \ + " admin_level INTEGER," \ + " housenumber TEXT," \ + " street TEXT," \ + " addr_place TEXT," \ + " isin TEXT," \ + " postcode TEXT," \ + " country_code VARCHAR(2)," \ + " extratags HSTORE" \ + ") %s %s" + +#define ADMINLEVEL_NONE 100 + +#define CREATE_PLACE_ID_INDEX \ + "CREATE INDEX place_id_idx ON place USING BTREE (osm_type, osm_id) %s %s" + + +enum { BUFFER_SIZE = 4092 }; + +void place_tag_processor::clear() +{ + // set members to sane defaults + src = NULL; + admin_level = ADMINLEVEL_NONE; + countrycode = 0; + housenumber.assign("\\N"); + street = 0; + addr_place = 0; + postcode = 0; + + places.clear(); + extratags.clear(); + address.clear(); + names.clear(); +} + +struct UnnamedPredicate +{ + bool operator()(const tag &val) const { + return val.key == "natural" || + val.key == "railway" || + val.key == "waterway" || + val.key == "boundary" || + (val.key == "highway" && + (val.value == "traffic_signals" || + val.value == "service" || + val.value == "cycleway" || + val.value == "path" || + val.value == "footway" || + val.value == "steps" || + val.value == "bridleway" || + val.value == "track" || + val.value == "byway" || + boost::ends_with(val.value, "_link"))); + } +}; + +void place_tag_processor::process_tags(const taglist_t &tags) +{ + bool placeadmin = false; + bool placehouse = false; + bool placebuilding = false; + const tag *place = 0; + const tag *junction = 0; + const tag *landuse = 0; + bool isnamed = false; + bool isinterpolation = false; + const std::string *house_nr = 0; + const std::string *conscr_nr = 0; + const std::string *street_nr = 0; + + clear(); + src = &tags; + + for (taglist_t::const_iterator item = tags.begin(); item != tags.end(); ++item) { + if (item->key == "name:prefix") { + extratags.push_back(&*item); + } else if (item->key == "ref" || + item->key == "int_ref" || + item->key == "nat_ref" || + item->key == "reg_ref" || + item->key == "loc_ref" || + item->key == "old_ref" || + item->key == "iata" || + item->key == "icao" || + item->key == "operator" || + item->key == "pcode" || + boost::starts_with(item->key, "pcode:")) { + names.push_back(&*item); + } else if (item->key == "name" || + boost::starts_with(item->key, "name:") || + item->key == "int_name" || + boost::starts_with(item->key, "int_name:") || + item->key == "nat_name" || + boost::starts_with(item->key, "nat_name:") || + item->key == "reg_name" || + boost::starts_with(item->key, "reg_name:") || + item->key == "loc_name" || + boost::starts_with(item->key, "loc_name:") || + item->key == "old_name" || + boost::starts_with(item->key, "old_name:") || + item->key == "alt_name" || + boost::starts_with(item->key, "alt_name:") || + boost::starts_with(item->key, "alt_name_") || + item->key == "official_name" || + boost::starts_with(item->key, "official_name:") || + item->key == "place_name" || + boost::starts_with(item->key, "place_name:") || + item->key == "short_name" || + boost::starts_with(item->key, "short_name:") || + item->key == "brand") { + names.push_back(&*item); + isnamed = true; + } else if (item->key == "addr:housename") { + names.push_back(&*item); + placehouse = true; + } else if (item->key == "emergency") { + if (item->value != "fire_hydrant" && + item->value != "yes" && + item->value != "no") + places.push_back(*item); + } else if (item->key == "tourism" || + item->key == "historic" || + item->key == "military") { + if (item->value != "no" && item->value != "yes") + places.push_back(*item); + } else if (item->key == "natural") { + if (item->value != "no" && + item->value != "yes" && + item->value != "coastline") + places.push_back(*item); + } else if (item->key == "landuse") { + if (item->value == "cemetry") + places.push_back(*item); + else + landuse = &*item; + } else if (item->key == "highway") { + if (item->value != "no" && + item->value != "turning_circle" && + item->value != "mini_roundabout" && + item->value != "noexit" && + item->value != "crossing") + places.push_back(*item); + } else if (item->key == "railway") { + if (item->value != "level_crossing" && + item->value != "no") + places.push_back(*item); + } else if (item->key == "man_made") { + if (item->value != "survey_point" && + item->value != "cutline") + places.push_back(*item); + } else if (item->key == "aerialway") { + if (item->value != "pylon" && + item->value != "no") + places.push_back(*item); + } else if (item->key == "boundary") { + if (item->value == "administrative") + placeadmin = true; + places.push_back(*item); + } else if (item->key == "aeroway" || + item->key == "amenity" || + item->key == "boundary" || + item->key == "bridge" || + item->key == "craft" || + item->key == "leisure" || + item->key == "office" || + item->key == "shop" || + item->key == "tunnel" || + item->key == "mountain_pass") { + if (item->value != "no") + { + places.push_back(*item); + } + } else if (item->key == "waterway") { + if (item->value != "riverbank") + places.push_back(*item); + } else if (item->key == "place") { + place = &*item; + } else if (item->key == "junction") { + junction = &*item; + } else if (item->key == "addr:interpolation") { + housenumber.clear(); + escape(item->value, housenumber); + isinterpolation = true; + } else if (item->key == "addr:housenumber") { + house_nr = &item->value; + placehouse = true; + } else if (item->key == "addr:conscriptionnumber") { + conscr_nr = &item->value; + placehouse = true; + } else if (item->key == "addr:streetnumber") { + street_nr = &item->value; + placehouse = true; + } else if (item->key == "addr:street") { + street = &item->value; + } else if (item->key == "addr:place") { + addr_place = &item->value; + } else if (item->key == "postal_code" || + item->key == "postcode" || + item->key == "addr:postcode" || + item->key == "tiger:zip_left" || + item->key == "tiger:zip_right") { + if (!postcode) + postcode = &item->value; + } else if (item->key == "country_code" || + item->key == "ISO3166-1" || + item->key == "is_in:country_code" || + item->key == "addr:country" || + item->key == "addr:country_code") { + if (item->value.length() == 2) + countrycode = &item->value; + } else if (boost::starts_with(item->key, "addr:") || + item->key == "is_in" || + boost::starts_with(item->key, "is_in:") || + item->key == "tiger:county") { + address.push_back(&*item); + } else if (item->key == "admin_level") { + admin_level = atoi(item->value.c_str()); + if (admin_level <= 0 || admin_level > 100) + admin_level = 100; + } else if (item->key == "tracktype" || + item->key == "traffic_calming" || + item->key == "service" || + item->key == "cuisine" || + item->key == "capital" || + item->key == "dispensing" || + item->key == "religion" || + item->key == "denomination" || + item->key == "sport" || + item->key == "internet_access" || + item->key == "lanes" || + item->key == "surface" || + item->key == "smoothness" || + item->key == "width" || + item->key == "est_width" || + item->key == "incline" || + item->key == "opening_hours" || + item->key == "collection_times" || + item->key == "service_times" || + item->key == "disused" || + item->key == "wheelchair" || + item->key == "sac_scale" || + item->key == "trail_visibility" || + item->key == "mtb:scale" || + item->key == "mtb:description" || + item->key == "wood" || + item->key == "drive_through" || + item->key == "drive_in" || + item->key == "access" || + item->key == "vehicle" || + item->key == "bicyle" || + item->key == "foot" || + item->key == "goods" || + item->key == "hgv" || + item->key == "motor_vehicle" || + item->key == "motor_car" || + boost::starts_with(item->key, "access:") || + boost::starts_with(item->key, "contact:") || + boost::starts_with(item->key, "drink:") || + item->key == "oneway" || + item->key == "date_on" || + item->key == "date_off" || + item->key == "day_on" || + item->key == "day_off" || + item->key == "hour_on" || + item->key == "hour_off" || + item->key == "maxweight" || + item->key == "maxheight" || + item->key == "maxspeed" || + item->key == "fee" || + item->key == "toll" || + boost::starts_with(item->key, "toll:") || + item->key == "charge" || + item->key == "population" || + item->key == "description" || + item->key == "image" || + item->key == "attribution" || + item->key == "fax" || + item->key == "email" || + item->key == "url" || + item->key == "website" || + item->key == "phone" || + item->key == "real_ale" || + item->key == "smoking" || + item->key == "food" || + item->key == "camera" || + item->key == "brewery" || + item->key == "locality" || + item->key == "wikipedia" || + boost::starts_with(item->key, "wikipedia:")) { + extratags.push_back(&*item); + } else if (item->key == "building") { + placebuilding = true; + } + } + + // skip some tags, if they don't have a proper name (ref doesn't count) + if (!isnamed) { + if (!places.empty()) + places.erase(std::remove_if(places.begin(), places.end(), + UnnamedPredicate()), + places.end()); + } + + if (isinterpolation) + places.push_back(tag("place", "houses")); + + if (place) { + if (isinterpolation || + (placeadmin && + place ->value != "island" && + place ->value != "islet")) + extratags.push_back(place); + else + places.push_back(*place); + } + + if (isnamed && places.empty()) { + if (junction) + places.push_back(*junction); + else if (landuse) + places.push_back(*landuse); + } + + if (places.empty()) { + if (placebuilding && (!names.empty() || placehouse || postcode)) { + places.push_back(tag("building", "yes")); + } else if (placehouse) { + places.push_back(tag("place", "house")); + } else if (postcode) { + places.push_back(tag("place", "postcode")); + } + } + + // housenumbers + if (!isinterpolation) { + if (street_nr && conscr_nr) { + housenumber.clear(); + escape(*conscr_nr, housenumber); + housenumber.append("/"); + escape(*street_nr, housenumber); + } else if (conscr_nr) { + housenumber.clear(); + escape(*conscr_nr, housenumber); + } else if (street_nr) { + housenumber.clear(); + escape(*street_nr, housenumber); + } else if (house_nr) { + housenumber.clear(); + escape(*house_nr, housenumber); + } + } + +} + +void place_tag_processor::copy_out(char osm_type, osmid_t osm_id, + const std::string &wkt, + std::string &buffer) +{ + BOOST_FOREACH(const tag &place, places) { + std::string name; + if (place.key == "bridge" || place.key == "tunnel") { + name = domain_name(place.key); + if (name.empty()) + continue; // don't include unnamed bridges and tunnels + } + + // osm_type + buffer += osm_type; + buffer += '\t'; + // osm_id + buffer += (single_fmt % osm_id).str(); + // class + escape(place.key, buffer); + buffer += '\t'; + // type + escape(place.value, buffer); + buffer += '\t'; + // names + if (!name.empty()) { + buffer += name; + buffer += '\t'; + } else if (!names.empty()) { + bool first = true; + // operator will be ignored on anything but these classes + // (amenity for restaurant and fuel) + bool shop = (place.key == "shop") || + (place.key == "amenity") || + (place.key == "tourism"); + BOOST_FOREACH(const tag *entry, names) { + if (!shop && (entry->key == "operator")) + continue; + + if (first) + first = false; + else + buffer += ','; + + buffer += "\""; + escape_array_record(entry->key, buffer); + buffer += "\"=>\""; + escape_array_record(entry->value, buffer); + buffer += "\""; + } + buffer += '\t'; + } else + buffer += "\\N\t"; + // admin_level + buffer += (single_fmt % admin_level).str(); + // house number + buffer += housenumber; + buffer += '\t'; + // street + copy_opt_string(street, buffer); + // addr_place + copy_opt_string(addr_place, buffer); + // isin + if (!address.empty()) { + BOOST_FOREACH(const tag *entry, address) { + if (entry->key == "tiger:county") { + escape(std::string(entry->value, 0, entry->value.find(",")), + buffer); + buffer += " county"; + } else { + escape(entry->value, buffer); + } + buffer += ','; + } + buffer[buffer.length() - 1] = '\t'; + } else + buffer += "\\N\t"; + // postcode + copy_opt_string(postcode, buffer); + // country code + copy_opt_string(countrycode, buffer); + // extra tags + if (extratags.empty()) { + buffer += "\\N\t"; + } else { + bool first = true; + BOOST_FOREACH(const tag *entry, extratags) { + if (first) + first = false; + else + buffer += ','; + + buffer += "\""; + escape_array_record(entry->key, buffer); + buffer += "\"=>\""; + escape_array_record(entry->value, buffer); + buffer += "\""; + } + buffer += "\t"; + } + // wkt + buffer += srid_str; + buffer += wkt; + buffer += '\n'; + } +} + + +void output_gazetteer_t::stop_copy(void) +{ + /* Do we have a copy active? */ + if (!copy_active) return; + + if (buffer.length() > 0) + { + pgsql_CopyData("place", Connection, buffer); + buffer.clear(); + } + + /* Terminate the copy */ + if (PQputCopyEnd(Connection, NULL) != 1) + { + std::cerr << "COPY_END for place failed: " << PQerrorMessage(Connection) << "\n"; + util::exit_nicely(); + } + + /* Check the result */ + PGresult *res = PQgetResult(Connection); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + std::cerr << "COPY_END for place failed: " << PQerrorMessage(Connection) << "\n"; + PQclear(res); + util::exit_nicely(); + } + + /* Discard the result */ + PQclear(res); + + /* We no longer have an active copy */ + copy_active = false; +} + + + +void output_gazetteer_t::delete_unused_classes(char osm_type, osmid_t osm_id) { + char tmp2[2]; + tmp2[0] = osm_type; tmp2[1] = '\0'; + char const *paramValues[2]; + paramValues[0] = tmp2; + paramValues[1] = (single_fmt % osm_id).str().c_str(); + PGresult *res = pgsql_execPrepared(ConnectionDelete, "get_classes", 2, + paramValues, PGRES_TUPLES_OK); + + int sz = PQntuples(res); + if (sz > 0 && !places.has_data()) { + PQclear(res); + /* unconditional delete of all places */ + delete_place(osm_type, osm_id); + } else { + std::string clslist; + for (int i = 0; i < sz; i++) { + std::string cls(PQgetvalue(res, i, 0)); + if (!places.has_place(cls)) { + clslist.reserve(clslist.length() + cls.length() + 3); + if (!clslist.empty()) + clslist += ','; + clslist += '\''; + clslist += cls; + clslist += '\''; + } + } + + PQclear(res); + + if (!clslist.empty()) { + /* Stop any active copy */ + stop_copy(); + + /* Delete all places for this object */ + pgsql_exec(Connection, PGRES_COMMAND_OK, + "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %" + PRIdOSMID " and class = any(ARRAY[%s])", + osm_type, osm_id, clslist.c_str()); + } + } +} + + +void output_gazetteer_t::delete_place(char osm_type, osmid_t osm_id) +{ + /* Stop any active copy */ + stop_copy(); + + /* Delete all places for this object */ + pgsql_exec(Connection, PGRES_COMMAND_OK, "DELETE FROM place WHERE osm_type = '%c' AND osm_id = %" PRIdOSMID, osm_type, osm_id); + + return; +} + +int output_gazetteer_t::connect() { + /* Connection to the database */ + Connection = PQconnectdb(m_options.conninfo.c_str()); + + /* Check to see that the backend connection was successfully made */ + if (PQstatus(Connection) != CONNECTION_OK) { + std::cerr << "Connection to database failed: " << PQerrorMessage(Connection) << "\n"; + return 1; + } + + if (m_options.append) { + ConnectionDelete = PQconnectdb(m_options.conninfo.c_str()); + if (PQstatus(ConnectionDelete) != CONNECTION_OK) + { + std::cerr << "Connection to database failed: " << PQerrorMessage(Connection) << "\n"; + return 1; + } + + pgsql_exec(ConnectionDelete, PGRES_COMMAND_OK, "PREPARE get_classes (CHAR(1), " POSTGRES_OSMID_TYPE ") AS SELECT class FROM place WHERE osm_type = $1 and osm_id = $2"); + } + return 0; +} + +int output_gazetteer_t::start() +{ + reproj = m_options.projection; + builder.set_exclude_broken_polygon(m_options.excludepoly); + + places.srid_str = (boost::format("SRID=%1%;") % SRID).str(); + + if(connect()) + util::exit_nicely(); + + /* Start a transaction */ + pgsql_exec(Connection, PGRES_COMMAND_OK, "BEGIN"); + + /* (Re)create the table unless we are appending */ + if (!m_options.append) { + /* Drop any existing table */ + pgsql_exec(Connection, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS place"); + + /* Create the new table */ + if (m_options.tblsmain_data) { + pgsql_exec(Connection, PGRES_COMMAND_OK, + CREATE_PLACE_TABLE, "TABLESPACE", m_options.tblsmain_data->c_str()); + } else { + pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_TABLE, "", ""); + } + if (m_options.tblsmain_index) { + pgsql_exec(Connection, PGRES_COMMAND_OK, + CREATE_PLACE_ID_INDEX, "TABLESPACE", m_options.tblsmain_index->c_str()); + } else { + pgsql_exec(Connection, PGRES_COMMAND_OK, CREATE_PLACE_ID_INDEX, "", ""); + } + + pgsql_exec(Connection, PGRES_TUPLES_OK, "SELECT AddGeometryColumn('place', 'geometry', %d, 'GEOMETRY', 2)", SRID); + pgsql_exec(Connection, PGRES_COMMAND_OK, "ALTER TABLE place ALTER COLUMN geometry SET NOT NULL"); + } + + return 0; +} + + +void output_gazetteer_t::stop() +{ + /* Stop any active copy */ + stop_copy(); + + /* Commit transaction */ + pgsql_exec(Connection, PGRES_COMMAND_OK, "COMMIT"); + + + PQfinish(Connection); + if (ConnectionDelete) + PQfinish(ConnectionDelete); + if (ConnectionError) + PQfinish(ConnectionError); + + return; +} + +int output_gazetteer_t::process_node(osmid_t id, double lat, double lon, + const taglist_t &tags) +{ + places.process_tags(tags); + + if (m_options.append) + delete_unused_classes('N', id); + + /* Are we interested in this item? */ + if (places.has_data()) { + std::string wkt = (point_fmt % lon % lat).str(); + places.copy_out('N', id, wkt, buffer); + flush_place_buffer(); + } + + return 0; +} + +int output_gazetteer_t::process_way(osmid_t id, const idlist_t &nds, const taglist_t &tags) +{ + places.process_tags(tags); + + if (m_options.append) + delete_unused_classes('W', id); + + /* Are we interested in this item? */ + if (places.has_data()) { + /* Fetch the node details */ + nodelist_t nodes; + m_mid->nodes_get_list(nodes, nds); + + /* Get the geometry of the object */ + geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodes, 1); + if (wkt) { + places.copy_out('W', id, wkt->geom, buffer); + flush_place_buffer(); + } + } + + return 0; +} + +int output_gazetteer_t::process_relation(osmid_t id, const memberlist_t &members, + const taglist_t &tags) +{ + const std::string *type = tags.get("type"); + if (!type) { + delete_unused_full('R', id); + return 0; + } + + int cmp_waterway = type->compare("waterway"); + + if (*type == "associatedStreet" + || !(*type == "boundary" || *type == "multipolygon" || !cmp_waterway)) { + delete_unused_full('R', id); + return 0; + } + + places.process_tags(tags); + + if (m_options.append) + delete_unused_classes('R', id); + + /* Are we interested in this item? */ + if (!places.has_data()) + return 0; + + /* get the boundary path (ways) */ + idlist_t xid2; + for (memberlist_t::const_iterator it = members.begin(); it != members.end(); ++it) { + /* only interested in ways */ + if (it->type == OSMTYPE_WAY) + xid2.push_back(it->id); + } + + if (xid2.empty()) { + if (m_options.append) + delete_unused_full('R', id); + + return 0; + } + + multitaglist_t xtags; + multinodelist_t xnodes; + idlist_t xid; + m_mid->ways_get_list(xid2, xid, xtags, xnodes); + + if (cmp_waterway) { + geometry_builder::maybe_wkts_t wkts = builder.build_both(xnodes, 1, 1, 1000000, id); + for (geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt) { + if (boost::starts_with(wkt->geom, "POLYGON") + || boost::starts_with(wkt->geom, "MULTIPOLYGON")) { + places.copy_out('R', id, wkt->geom, buffer); + flush_place_buffer(); + } else { + /* add_polygon_error('R', id, "boundary", "adminitrative", &names, countrycode, wkt); */ + } + } + } else { + /* waterways result in multilinestrings */ + geometry_builder::maybe_wkt_t wkt = builder.build_multilines(xnodes, id); + if ((wkt->geom).length() > 0) { + places.copy_out('R', id, wkt->geom, buffer); + flush_place_buffer(); + } + } + + return 0; +} + + diff -Nru osm2pgsql-0.82.0/output-gazetteer.h osm2pgsql-0.88.1/output-gazetteer.h --- osm2pgsql-0.82.0/output-gazetteer.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/output-gazetteer.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -#ifndef OUTPUT_GAZETTEER_H -#define OUTPUT_GAZETTEER_H - -#include "output.h" - -struct output_t out_gazetteer; - -#endif diff -Nru osm2pgsql-0.82.0/output-gazetteer.hpp osm2pgsql-0.88.1/output-gazetteer.hpp --- osm2pgsql-0.82.0/output-gazetteer.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-gazetteer.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,277 @@ +#ifndef OUTPUT_GAZETTEER_H +#define OUTPUT_GAZETTEER_H + +#include "osmtypes.hpp" +#include "output.hpp" +#include "geometry-builder.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include +#include +#include + +#include +#include + +/** + * A private class to convert tags. + */ +class place_tag_processor +{ +public: + place_tag_processor() + : single_fmt("%1%\t") + { + places.reserve(4); + extratags.reserve(15); + address.reserve(10); + } + + ~place_tag_processor() {} + + void process_tags(const taglist_t &tags); + + bool has_data() const { return !places.empty(); } + + bool has_place(const std::string &cls) + { + BOOST_FOREACH(const tag &item, places) { + if (cls == item.key) + return true; + } + + return false; + } + + void copy_out(char osm_type, osmid_t osm_id, const std::string &wkt, + std::string &buffer); + + void clear(); + +private: + void copy_opt_string(const std::string *val, std::string &buffer) + { + if (val) { + escape(*val, buffer); + buffer += "\t"; + } else { + buffer += "\\N\t"; + } + } + + std::string domain_name(const std::string &cls) + { + std::string ret; + bool hasname = false; + + std::string prefix(cls + ":name"); + + for (taglist_t::const_iterator item = src->begin(); item != src->end(); ++item) { + if (boost::starts_with(item->key, prefix) && + (item->key.length() == prefix.length() + || item->key[prefix.length()] == ':')) { + if (!hasname) { + ret.reserve(item->key.length() + item->value.length() + 10); + hasname = true; + } else + ret += ","; + ret += "\""; + escape_array_record(std::string(item->key, cls.length() + 1), ret); + ret += "\"=>\""; + escape_array_record(item->value, ret); + ret += "\""; + } + } + + return ret; + } + + + void escape_array_record(const std::string &in, std::string &out) + { + BOOST_FOREACH(const char c, in) { + switch(c) { + case '\\': out += "\\\\\\\\\\\\\\\\"; break; + case '\n': + case '\r': + case '\t': + case '"': + /* This is a bit naughty - we know that nominatim ignored these characters so just drop them now for simplicity */ + out += ' '; break; + default: out += c; break; + } + } + } + + + std::vector places; + std::vector names; + std::vector extratags; + std::vector address; + const taglist_t *src; + int admin_level; + const std::string *countrycode; + std::string housenumber; + const std::string *street; + const std::string *addr_place; + const std::string *postcode; + + boost::format single_fmt; +public: + std::string srid_str; +}; + + +class output_gazetteer_t : public output_t { +public: + output_gazetteer_t(const middle_query_t* mid_, const options_t &options_) + : output_t(mid_, options_), + Connection(NULL), + ConnectionDelete(NULL), + ConnectionError(NULL), + copy_active(false), + single_fmt("%1%"), + point_fmt("POINT(%.15g %.15g)") + { + buffer.reserve(PLACE_BUFFER_SIZE); + } + + output_gazetteer_t(const output_gazetteer_t& other) + : output_t(other.m_mid, other.m_options), + Connection(NULL), + ConnectionDelete(NULL), + ConnectionError(NULL), + copy_active(false), + reproj(other.reproj), + single_fmt(other.single_fmt), + point_fmt(other.point_fmt) + { + buffer.reserve(PLACE_BUFFER_SIZE); + builder.set_exclude_broken_polygon(m_options.excludepoly); + connect(); + } + + virtual ~output_gazetteer_t() {} + + virtual boost::shared_ptr clone(const middle_query_t* cloned_middle) const + { + output_gazetteer_t *clone = new output_gazetteer_t(*this); + clone->m_mid = cloned_middle; + return boost::shared_ptr(clone); + } + + int start(); + void stop(); + void commit() {} + + void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {} + int pending_way(osmid_t id, int exists) { return 0; } + + void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) {} + int pending_relation(osmid_t id, int exists) { return 0; } + + int node_add(osmid_t id, double lat, double lon, const taglist_t &tags) + { + return process_node(id, lat, lon, tags); + } + + int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) + { + return process_way(id, nodes, tags); + } + + int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) + { + return process_relation(id, members, tags); + } + + int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) + { + return process_node(id, lat, lon, tags); + } + + int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) + { + return process_way(id, nodes, tags); + } + + int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) + { + return process_relation(id, members, tags); + } + + int node_delete(osmid_t id) + { + delete_place('N', id); + return 0; + } + + int way_delete(osmid_t id) + { + delete_place('W', id); + return 0; + } + + int relation_delete(osmid_t id) + { + delete_place('R', id); + return 0; + } + +private: + enum { PLACE_BUFFER_SIZE = 4092 }; + + void stop_copy(void); + void delete_unused_classes(char osm_type, osmid_t osm_id); + void delete_place(char osm_type, osmid_t osm_id); + int process_node(osmid_t id, double lat, double lon, const taglist_t &tags); + int process_way(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int process_relation(osmid_t id, const memberlist_t &members, const taglist_t &tags); + int connect(); + + void flush_place_buffer() + { + if (!copy_active) + { + pgsql_exec(Connection, PGRES_COPY_IN, "COPY place (osm_type, osm_id, class, type, name, admin_level, housenumber, street, addr_place, isin, postcode, country_code, extratags, geometry) FROM STDIN"); + copy_active = true; + } + + pgsql_CopyData("place", Connection, buffer); + buffer.clear(); + } + + void delete_unused_full(char osm_type, osmid_t osm_id) + { + if (m_options.append) { + places.clear(); + delete_place(osm_type, osm_id); + } + } + + struct pg_conn *Connection; + struct pg_conn *ConnectionDelete; + struct pg_conn *ConnectionError; + + bool copy_active; + + std::string buffer; + place_tag_processor places; + + geometry_builder builder; + + boost::shared_ptr reproj; + + // string formatters + // Need to be part of the class, so we have one per thread. + boost::format single_fmt; + boost::format point_fmt; + + const static std::string NAME; +}; + +extern output_gazetteer_t out_gazetteer; + +#endif diff -Nru osm2pgsql-0.82.0/output.h osm2pgsql-0.88.1/output.h --- osm2pgsql-0.82.0/output.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/output.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,81 +0,0 @@ -/* Common output layer interface */ - -/* Each output layer must provide methods for - * storing: - * - Nodes (Points of interest etc) - * - Way geometries - * Associated tags: name, type etc. -*/ - -#ifndef OUTPUT_H -#define OUTPUT_H - -#include "middle.h" -#include "keyvals.h" - -/* Variants for generation of hstore column */ -/* No hstore column */ -#define HSTORE_NONE 0 -/* create a hstore column for all tags which do not have an exclusive column */ -#define HSTORE_NORM 1 -/* create a hstore column for all tags */ -#define HSTORE_ALL 2 - -struct output_options { - const char *conninfo; /* Connection info string */ - const char *prefix; /* prefix for table names */ - int scale; /* scale for converting coordinates to fixed point */ - int projection; /* SRS of projection */ - int append; /* Append to existing data */ - int slim; /* In slim mode */ - int cache; /* Memory usable for cache in MB */ - struct middle_t *mid; /* Mid storage to use */ - struct output_t *out; /* Output type used */ - const char *tblsmain_index; /* Pg Tablespace to store indexes on main tables */ - const char *tblsslim_index; /* Pg Tablespace to store indexes on slim tables */ - const char *tblsmain_data; /* Pg Tablespace to store main tables */ - const char *tblsslim_data; /* Pg Tablespace to store slim tables */ - const char *style; /* style file to use */ - int expire_tiles_zoom; /* Zoom level for tile expiry list */ - int expire_tiles_zoom_min; /* Minimum zoom level for tile expiry list */ - const char *expire_tiles_filename; /* File name to output expired tiles list to */ - int enable_hstore; /* add an additional hstore column with objects key/value pairs */ - int enable_hstore_index; /* add an index on the hstore column */ - int enable_multi; /* Output multi-geometries intead of several simple geometries */ - const char** hstore_columns; /* list of columns that should be written into their own hstore column */ - int n_hstore_columns; /* number of hstore columns */ - int keep_coastlines; - int parallel_indexing; - int alloc_chunkwise; - int num_procs; - int droptemp; /* drop slim mode temp tables after act */ - int unlogged; /* use unlogged tables where possible */ - int hstore_match_only; /* only copy rows that match an explicitly listed key */ - int flat_node_cache_enabled; - int excludepoly; - const char *flat_node_file; -}; - -struct output_t { - int (*start)(const struct output_options *options); - int (*connect)(const struct output_options *options, int startTransaction); - void (*stop)(); - void (*cleanup)(void); - void (*close)(int stopTransaction); - - int (*node_add)(osmid_t id, double lat, double lon, struct keyval *tags); - int (*way_add)(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags); - int (*relation_add)(osmid_t id, struct member *members, int member_count, struct keyval *tags); - - int (*node_modify)(osmid_t id, double lat, double lon, struct keyval *tags); - int (*way_modify)(osmid_t id, osmid_t *nodes, int node_count, struct keyval *tags); - int (*relation_modify)(osmid_t id, struct member *members, int member_count, struct keyval *tags); - - int (*node_delete)(osmid_t id); - int (*way_delete)(osmid_t id); - int (*relation_delete)(osmid_t id); -}; - -unsigned int pgsql_filter_tags(enum OsmType type, struct keyval *tags, int *polygon); - -#endif diff -Nru osm2pgsql-0.82.0/output.hpp osm2pgsql-0.88.1/output.hpp --- osm2pgsql-0.82.0/output.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,79 @@ +/* Common output layer interface */ + +/* Each output layer must provide methods for + * storing: + * - Nodes (Points of interest etc) + * - Way geometries + * Associated tags: name, type etc. +*/ + +#ifndef OUTPUT_H +#define OUTPUT_H + +#include "options.hpp" +#include "middle.hpp" +#include "id-tracker.hpp" +#include "expire-tiles.hpp" + +#include +#include +#include +#include + +struct pending_job_t { + osmid_t osm_id; + size_t output_id; + + pending_job_t() : osm_id(0), output_id(0) {} + pending_job_t(osmid_t id, size_t oid) : osm_id(id), output_id(oid) {} +}; + +typedef std::stack pending_queue_t; + +class output_t : public boost::noncopyable { +public: + static std::vector > create_outputs(const middle_query_t *mid, const options_t &options); + + output_t(const middle_query_t *mid, const options_t &options_); + virtual ~output_t(); + + virtual boost::shared_ptr clone(const middle_query_t* cloned_middle) const = 0; + + virtual int start() = 0; + virtual void stop() = 0; + virtual void commit() = 0; + + virtual void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) = 0; + virtual int pending_way(osmid_t id, int exists) = 0; + + virtual void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) = 0; + virtual int pending_relation(osmid_t id, int exists) = 0; + + virtual int node_add(osmid_t id, double lat, double lon, const taglist_t &tags) = 0; + virtual int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) = 0; + virtual int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) = 0; + + virtual int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) = 0; + virtual int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) = 0; + virtual int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) = 0; + + virtual int node_delete(osmid_t id) = 0; + virtual int way_delete(osmid_t id) = 0; + virtual int relation_delete(osmid_t id) = 0; + + virtual size_t pending_count() const; + + const options_t *get_options() const; + + virtual void merge_pending_relations(boost::shared_ptr other); + virtual void merge_expire_trees(boost::shared_ptr other); + virtual boost::shared_ptr get_pending_relations(); + virtual boost::shared_ptr get_expire_tree(); + +protected: + + const middle_query_t* m_mid; + const options_t m_options; +}; + +#endif diff -Nru osm2pgsql-0.82.0/output-multi.cpp osm2pgsql-0.88.1/output-multi.cpp --- osm2pgsql-0.82.0/output-multi.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-multi.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,451 @@ +#include "output-multi.hpp" +#include "taginfo_impl.hpp" +#include "table.hpp" +#include "tagtransform.hpp" +#include "options.hpp" +#include "middle.hpp" +#include "id-tracker.hpp" +#include "geometry-builder.hpp" +#include "expire-tiles.hpp" + +#include +#include + +output_multi_t::output_multi_t(const std::string &name, + boost::shared_ptr processor_, + const struct export_list &export_list_, + const middle_query_t* mid_, const options_t &options_) + : output_t(mid_, options_), + m_tagtransform(new tagtransform(&m_options)), + m_export_list(new export_list(export_list_)), + m_processor(processor_), + //TODO: we could in fact have something that is interested in nodes and ways.. + m_osm_type(m_processor->interests(geometry_processor::interest_node) ? OSMTYPE_NODE : OSMTYPE_WAY), + m_table(new table_t(m_options.conninfo, name, m_processor->column_type(), + m_export_list->normal_columns(m_osm_type), + m_options.hstore_columns, m_processor->srid(), + m_options.append, m_options.slim, m_options.droptemp, + m_options.hstore_mode, m_options.enable_hstore_index, + m_options.tblsmain_data, m_options.tblsmain_index)), + ways_pending_tracker(new id_tracker()), ways_done_tracker(new id_tracker()), rels_pending_tracker(new id_tracker()), + m_expire(new expire_tiles(&m_options)) { +} + +output_multi_t::output_multi_t(const output_multi_t& other): + output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_export_list(new export_list(*other.m_export_list)), + m_processor(other.m_processor), m_osm_type(other.m_osm_type), m_table(new table_t(*other.m_table)), + ways_pending_tracker(new id_tracker()), ways_done_tracker(new id_tracker()), rels_pending_tracker(new id_tracker()), + m_expire(new expire_tiles(&m_options)) { +} + + +output_multi_t::~output_multi_t() { +} + +boost::shared_ptr output_multi_t::clone(const middle_query_t* cloned_middle) const{ + output_multi_t *clone = new output_multi_t(*this); + clone->m_mid = cloned_middle; + //NOTE: we need to know which ways were used by relations so each thread + //must have a copy of the original marked done ways, its read only so its ok + clone->ways_done_tracker = ways_done_tracker; + return boost::shared_ptr(clone); +} + +int output_multi_t::start() { + m_table->start(); + return 0; +} + +size_t output_multi_t::pending_count() const { + return ways_pending_tracker->size() + rels_pending_tracker->size(); +} + +void output_multi_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { + osmid_t const prev = ways_pending_tracker->last_returned(); + if (id_tracker::is_valid(prev) && prev >= id) { + if (prev > id) { + job_queue.push(pending_job_t(id, output_id)); + } + // already done the job + return; + } + + //make sure we get the one passed in + if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) { + job_queue.push(pending_job_t(id, output_id)); + added++; + } + + //grab the first one or bail if its not valid + osmid_t popped = ways_pending_tracker->pop_mark(); + if(!id_tracker::is_valid(popped)) + return; + + //get all the ones up to the id that was passed in + while (popped < id) { + if (!ways_done_tracker->is_marked(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + popped = ways_pending_tracker->pop_mark(); + } + + //make sure to get this one as well and move to the next + if(popped == id) { + if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + } +} + +int output_multi_t::pending_way(osmid_t id, int exists) { + taglist_t tags_int; + nodelist_t nodes_int; + int ret = 0; + + // Try to fetch the way from the DB + if (!m_mid->ways_get(id, tags_int, nodes_int)) { + // Output the way + ret = reprocess_way(id, nodes_int, tags_int, exists); + } + + return ret; +} + +void output_multi_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { + osmid_t const prev = rels_pending_tracker->last_returned(); + if (id_tracker::is_valid(prev) && prev >= id) { + if (prev > id) { + job_queue.push(pending_job_t(id, output_id)); + } + // already done the job + return; + } + + //make sure we get the one passed in + if(id_tracker::is_valid(id)) { + job_queue.push(pending_job_t(id, output_id)); + added++; + } + + //grab the first one or bail if its not valid + osmid_t popped = rels_pending_tracker->pop_mark(); + if(!id_tracker::is_valid(popped)) + return; + + //get all the ones up to the id that was passed in + while (popped < id) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + popped = rels_pending_tracker->pop_mark(); + } + + //make sure to get this one as well and move to the next + if(popped == id) { + if(id_tracker::is_valid(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + } +} + +int output_multi_t::pending_relation(osmid_t id, int exists) { + taglist_t tags_int; + memberlist_t members_int; + int ret = 0; + + // Try to fetch the relation from the DB + if (!m_mid->relations_get(id, members_int, tags_int)) { + ret = process_relation(id, members_int, tags_int, exists); + } + + return ret; +} + +void output_multi_t::stop() { + m_table->stop(); + m_expire->output_and_destroy(); + m_expire.reset(); +} + +void output_multi_t::commit() { + m_table->commit(); +} + +int output_multi_t::node_add(osmid_t id, double lat, double lon, const taglist_t &tags) { + if (m_processor->interests(geometry_processor::interest_node)) { + return process_node(id, lat, lon, tags); + } + return 0; +} + +int output_multi_t::way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags) { + if (m_processor->interests(geometry_processor::interest_way) && nodes.size() > 1) { + return process_way(id, nodes, tags); + } + return 0; +} + + +int output_multi_t::relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) { + if (m_processor->interests(geometry_processor::interest_relation) && !members.empty()) { + return process_relation(id, members, tags, 0); + } + return 0; +} + +int output_multi_t::node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) { + if (m_processor->interests(geometry_processor::interest_node)) { + // TODO - need to know it's a node? + delete_from_output(id); + + // TODO: need to mark any ways or relations using it - depends on what + // type of output this is... delegate to the geometry processor?? + return process_node(id, lat, lon, tags); + + } else { + return 0; + } +} + +int output_multi_t::way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags) { + if (m_processor->interests(geometry_processor::interest_way)) { + // TODO - need to know it's a way? + delete_from_output(id); + + // TODO: need to mark any relations using it - depends on what + // type of output this is... delegate to the geometry processor?? + return process_way(id, nodes, tags); + + } else { + return 0; + } +} + +int output_multi_t::relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) { + if (m_processor->interests(geometry_processor::interest_relation)) { + // TODO - need to know it's a relation? + delete_from_output(-id); + + // TODO: need to mark any other relations using it - depends on what + // type of output this is... delegate to the geometry processor?? + return process_relation(id, members, tags, false); + + } else { + return 0; + } +} + +int output_multi_t::node_delete(osmid_t id) { + if (m_processor->interests(geometry_processor::interest_node)) { + // TODO - need to know it's a node? + delete_from_output(id); + } + return 0; +} + +int output_multi_t::way_delete(osmid_t id) { + if (m_processor->interests(geometry_processor::interest_way)) { + // TODO - need to know it's a way? + delete_from_output(id); + } + return 0; +} + +int output_multi_t::relation_delete(osmid_t id) { + if (m_processor->interests(geometry_processor::interest_relation)) { + // TODO - need to know it's a relation? + delete_from_output(-id); + } + return 0; +} + +int output_multi_t::process_node(osmid_t id, double lat, double lon, const taglist_t &tags) { + //check if we are keeping this node + taglist_t outtags; + unsigned int filter = m_tagtransform->filter_node_tags(tags, *m_export_list.get(), outtags, true); + if (!filter) { + //grab its geom + geometry_builder::maybe_wkt_t wkt = m_processor->process_node(lat, lon); + if (wkt) { + m_expire->from_bbox(lon, lat, lon, lat); + copy_to_table(id, wkt->geom.c_str(), outtags); + } + } + return 0; +} + +int output_multi_t::reprocess_way(osmid_t id, const nodelist_t &nodes, const taglist_t &tags, bool exists) +{ + //if the way could exist already we have to make the relation pending and reprocess it later + //but only if we actually care about relations + if(m_processor->interests(geometry_processor::interest_relation) && exists) { + way_delete(id); + const std::vector rel_ids = m_mid->relations_using_way(id); + for (std::vector::const_iterator itr = rel_ids.begin(); itr != rel_ids.end(); ++itr) { + rels_pending_tracker->mark(*itr); + } + } + + //check if we are keeping this way + int polygon = 0, roads = 0; + taglist_t outtags; + unsigned int filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads, + *m_export_list.get(), outtags, true); + if (!filter) { + //grab its geom + geometry_builder::maybe_wkt_t wkt = m_processor->process_way(nodes); + if (wkt) { + //TODO: need to know if we care about polygons or lines for this output + //the difference only being that if its a really large bbox for the poly + //it downgrades to just invalidating the line/perimeter anyway + if(boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")) + m_expire->from_nodes_poly(nodes, id); + else + m_expire->from_nodes_line(nodes); + copy_to_table(id, wkt->geom.c_str(), outtags); + } + } + return 0; +} + +int output_multi_t::process_way(osmid_t id, const idlist_t &nodes, const taglist_t &tags) { + //check if we are keeping this way + int polygon = 0, roads = 0; + taglist_t outtags; + unsigned filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads, + *m_export_list.get(), outtags, true); + if (!filter) { + //get the geom from the middle + if(m_way_helper.set(nodes, m_mid) < 1) + return 0; + //grab its geom + geometry_builder::maybe_wkt_t wkt = m_processor->process_way(m_way_helper.node_cache); + + if (wkt) { + //if we are also interested in relations we need to mark + //this way pending just in case it shows up in one + if (m_processor->interests(geometry_processor::interest_relation)) { + ways_pending_tracker->mark(id); + }//we aren't interested in relations so if it comes in on a relation later we wont keep it + else { + //TODO: need to know if we care about polygons or lines for this output + //the difference only being that if its a really large bbox for the poly + //it downgrades to just invalidating the line/perimeter anyway + if(boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")) + m_expire->from_nodes_poly(m_way_helper.node_cache, id); + else + m_expire->from_nodes_line(m_way_helper.node_cache); + copy_to_table(id, wkt->geom.c_str(), outtags); + } + } + } + return 0; +} + +int output_multi_t::process_relation(osmid_t id, const memberlist_t &members, + const taglist_t &tags, bool exists, bool pending) { + //if it may exist already, delete it first + if(exists) + relation_delete(id); + + //does this relation have anything interesting to us + taglist_t rel_outtags; + unsigned filter = m_tagtransform->filter_rel_tags(tags, *m_export_list.get(), + rel_outtags, true); + if (!filter) { + //TODO: move this into geometry processor, figure a way to come back for tag transform + //grab ways/nodes of the members in the relation, bail if none were used + if(m_relation_helper.set(&members, (middle_t*)m_mid) < 1) + return 0; + + //filter the tags on each member because we got them from the middle + //and since the middle is no longer tied to the output it no longer + //shares any kind of tag transform and therefore has all original tags + //so we filter here because each individual outputs cares about different tags + int polygon, roads; + multitaglist_t filtered(m_relation_helper.tags.size(), taglist_t()); + for(size_t i = 0; i < m_relation_helper.tags.size(); ++i) + { + m_tagtransform->filter_way_tags(m_relation_helper.tags[i], &polygon, + &roads, *m_export_list.get(), filtered[i]); + //TODO: if the filter says that this member is now not interesting we + //should decrement the count and remove his nodes and tags etc. for + //now we'll just keep him with no tags so he will get filtered later + } + + //do the members of this relation have anything interesting to us + //NOTE: make_polygon is preset here this is to force the tag matching/superseeded stuff + //normally this wouldnt work but we tell the tag transform to allow typeless relations + //this is needed because the type can get stripped off by the rel_tag filter above + //if the export list did not include the type tag. + //TODO: find a less hacky way to do the matching/superseeded and tag copying stuff without + //all this trickery + int make_boundary, make_polygon = 1; + taglist_t outtags; + filter = m_tagtransform->filter_rel_member_tags(rel_outtags, filtered, m_relation_helper.roles, + &m_relation_helper.superseeded.front(), + &make_boundary, &make_polygon, &roads, + *m_export_list.get(), outtags, true); + if (!filter) + { + geometry_builder::maybe_wkts_t wkts = m_processor->process_relation(m_relation_helper.nodes); + if (wkts) { + for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt) + { + //TODO: we actually have the nodes in the m_relation_helper and could use them + //instead of having to reparse the wkt in the expiry code + m_expire->from_wkt(wkt->geom.c_str(), -id); + //what part of the code relies on relation members getting negative ids? + copy_to_table(-id, wkt->geom.c_str(), outtags); + } + } + + //TODO: should this loop be inside the if above just in case? + //take a look at each member to see if its superseeded (tags on it matched the tags on the relation) + for(size_t i = 0; i < m_relation_helper.ways.size(); ++i) { + //tags matched so we are keeping this one with this relation + if (m_relation_helper.superseeded[i]) { + //just in case it wasnt previously with this relation we get rid of them + way_delete(m_relation_helper.ways[i]); + //the other option is that we marked them pending in the way processing so here we mark them + //done so when we go back over the pendings we can just skip it because its in the done list + //TODO: dont do this when working with pending relations to avoid thread races + if(!pending) + ways_done_tracker->mark(m_relation_helper.ways[i]); + } + } + } + } + return 0; +} + +void output_multi_t::copy_to_table(osmid_t id, const char *wkt, const taglist_t &tags) { + m_table->write_wkt(id, tags, wkt); +} + +void output_multi_t::delete_from_output(osmid_t id) { + if(m_expire->from_db(m_table.get(), id)) + m_table->delete_row(id); +} + +void output_multi_t::merge_pending_relations(boost::shared_ptr other) { + boost::shared_ptr tracker = other.get()->get_pending_relations(); + osmid_t id; + while(tracker.get() && id_tracker::is_valid((id = tracker->pop_mark()))){ + rels_pending_tracker->mark(id); + } +} + +void output_multi_t::merge_expire_trees(boost::shared_ptr other) { + if(other->get_expire_tree().get()) + m_expire->merge_and_destroy(*other.get()->get_expire_tree()); +} + +boost::shared_ptr output_multi_t::get_pending_relations() { + return rels_pending_tracker; +} +boost::shared_ptr output_multi_t::get_expire_tree() { + return m_expire; +} diff -Nru osm2pgsql-0.82.0/output-multi.hpp osm2pgsql-0.88.1/output-multi.hpp --- osm2pgsql-0.82.0/output-multi.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-multi.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,89 @@ +/* One implementation of output-layer processing for osm2pgsql. + * Manages a single table, transforming geometry using a + * variety of algorithms plus tag transformation for the + * database columns. + */ + +#ifndef OUTPUT_MULTI_HPP +#define OUTPUT_MULTI_HPP + +#include "osmtypes.hpp" +#include "output.hpp" +#include "geometry-processor.hpp" + +#include +#include +#include +#include + +class table_t; +class tagtransform; +struct expire_tiles; +struct export_list; +struct id_tracker; +struct middle_query_t; +struct options_t; + +class output_multi_t : public output_t { +public: + output_multi_t(const std::string &name, + boost::shared_ptr processor_, + const export_list &export_list_, + const middle_query_t* mid_, const options_t &options_); + output_multi_t(const output_multi_t& other); + virtual ~output_multi_t(); + + virtual boost::shared_ptr clone(const middle_query_t* cloned_middle) const; + + int start(); + void stop(); + void commit(); + + void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added); + int pending_way(osmid_t id, int exists); + + void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added); + int pending_relation(osmid_t id, int exists); + + int node_add(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_delete(osmid_t id); + int way_delete(osmid_t id); + int relation_delete(osmid_t id); + + size_t pending_count() const; + + void merge_pending_relations(boost::shared_ptr other); + void merge_expire_trees(boost::shared_ptr other); + virtual boost::shared_ptr get_pending_relations(); + virtual boost::shared_ptr get_expire_tree(); + +protected: + + void delete_from_output(osmid_t id); + int process_node(osmid_t id, double lat, double lon, const taglist_t &tags); + int process_way(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int reprocess_way(osmid_t id, const nodelist_t &nodes, const taglist_t &tags, bool exists); + int process_relation(osmid_t id, const memberlist_t &members, const taglist_t &tags, bool exists, bool pending=false); + void copy_to_table(osmid_t id, const char *wkt, const taglist_t &tags); + + boost::scoped_ptr m_tagtransform; + boost::scoped_ptr m_export_list; + boost::shared_ptr m_processor; + const OsmType m_osm_type; + boost::scoped_ptr m_table; + boost::shared_ptr ways_pending_tracker, ways_done_tracker, rels_pending_tracker; + boost::shared_ptr m_expire; + way_helper m_way_helper; + relation_helper m_relation_helper; + + const static std::string NAME; +}; + +#endif diff -Nru osm2pgsql-0.82.0/output-null.c osm2pgsql-0.88.1/output-null.c --- osm2pgsql-0.82.0/output-null.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/output-null.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,75 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "osmtypes.h" -#include "output.h" -#include "output-null.h" - -#define UNUSED __attribute__ ((unused)) - -static void null_out_cleanup(void) { -} - -static int null_out_start(const struct output_options *opt UNUSED) { - return 0; -} - -static void null_out_stop() { -} - -static int null_add_node(osmid_t a UNUSED, double b UNUSED, double c UNUSED, struct keyval *k UNUSED) { - return 0; -} - -static int null_add_way(osmid_t a UNUSED, osmid_t *b UNUSED, int c UNUSED, struct keyval *k UNUSED) { - return 0; -} - -static int null_add_relation(osmid_t a UNUSED, struct member *b UNUSED, int c UNUSED, struct keyval *k UNUSED) { - return 0; -} - -static int null_delete_node(osmid_t i UNUSED) { - return 0; -} - -static int null_delete_way(osmid_t i UNUSED) { - return 0; -} - -static int null_delete_relation(osmid_t i UNUSED) { - return 0; -} - -static int null_modify_node(osmid_t a UNUSED, double b UNUSED, double c UNUSED, struct keyval * k UNUSED) { - return 0; -} - -static int null_modify_way(osmid_t a UNUSED, osmid_t * b UNUSED, int c UNUSED, struct keyval * k UNUSED) { - return 0; -} - -static int null_modify_relation(osmid_t a UNUSED, struct member * b UNUSED, int c UNUSED, struct keyval * k UNUSED) { - return 0; -} - -struct output_t out_null = { - .start = null_out_start, - .stop = null_out_stop, - .cleanup = null_out_cleanup, - .node_add = null_add_node, - .way_add = null_add_way, - .relation_add = null_add_relation, - - .node_modify = null_modify_node, - .way_modify = null_modify_way, - .relation_modify = null_modify_relation, - - .node_delete = null_delete_node, - .way_delete = null_delete_way, - .relation_delete = null_delete_relation -}; diff -Nru osm2pgsql-0.82.0/output-null.cpp osm2pgsql-0.88.1/output-null.cpp --- osm2pgsql-0.82.0/output-null.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-null.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,83 @@ +#include "osmtypes.hpp" +#include "output-null.hpp" + +struct middle_query_t; +struct options_t; + +void output_null_t::cleanup() { +} + +int output_null_t::start() { + return 0; +} + +void output_null_t::stop() { +} + +void output_null_t::commit() { +} + +void output_null_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { +} + +int output_null_t::pending_way(osmid_t id, int exists) { + return 0; +} + +void output_null_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { +} + +int output_null_t::pending_relation(osmid_t id, int exists) { + return 0; +} + +int output_null_t::node_add(osmid_t, double, double, const taglist_t &) { + return 0; +} + +int output_null_t::way_add(osmid_t a, const idlist_t &, const taglist_t &) { + return 0; +} + +int output_null_t::relation_add(osmid_t a, const memberlist_t &, const taglist_t &) { + return 0; +} + +int output_null_t::node_delete(osmid_t i) { + return 0; +} + +int output_null_t::way_delete(osmid_t i) { + return 0; +} + +int output_null_t::relation_delete(osmid_t i) { + return 0; +} + +int output_null_t::node_modify(osmid_t, double, double, const taglist_t &) { + return 0; +} + +int output_null_t::way_modify(osmid_t, const idlist_t &, const taglist_t &) { + return 0; +} + +int output_null_t::relation_modify(osmid_t, const memberlist_t &, const taglist_t &) { + return 0; +} + +boost::shared_ptr output_null_t::clone(const middle_query_t* cloned_middle) const { + output_null_t *clone = new output_null_t(*this); + clone->m_mid = cloned_middle; + return boost::shared_ptr(clone); +} + +output_null_t::output_null_t(const middle_query_t* mid_, const options_t &options_): output_t(mid_, options_) { +} + +output_null_t::output_null_t(const output_null_t& other): output_t(other.m_mid, other.m_options) { +} + +output_null_t::~output_null_t() { +} diff -Nru osm2pgsql-0.82.0/output-null.h osm2pgsql-0.88.1/output-null.h --- osm2pgsql-0.82.0/output-null.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/output-null.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -/* Implements dummy output-layer processing for testing. -*/ - -#ifndef OUTPUT_NULL_H -#define OUTPUT_NULL_H - -#include "output.h" - -extern struct output_t out_null; - -#endif diff -Nru osm2pgsql-0.82.0/output-null.hpp osm2pgsql-0.88.1/output-null.hpp --- osm2pgsql-0.82.0/output-null.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-null.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,41 @@ +/* Implements dummy output-layer processing for testing. +*/ + +#ifndef OUTPUT_NULL_H +#define OUTPUT_NULL_H + +#include "output.hpp" + +class output_null_t : public output_t { +public: + output_null_t(const middle_query_t* mid_, const options_t &options); + output_null_t(const output_null_t& other); + virtual ~output_null_t(); + + virtual boost::shared_ptr clone(const middle_query_t* cloned_middle) const; + + int start(); + void stop(); + void commit(); + void cleanup(void); + + void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added); + int pending_way(osmid_t id, int exists); + + void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added); + int pending_relation(osmid_t id, int exists); + + int node_add(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_delete(osmid_t id); + int way_delete(osmid_t id); + int relation_delete(osmid_t id); +}; + +#endif diff -Nru osm2pgsql-0.82.0/output-pgsql.c osm2pgsql-0.88.1/output-pgsql.c --- osm2pgsql-0.82.0/output-pgsql.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/output-pgsql.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,1887 +0,0 @@ -/* Implements the mid-layer processing for osm2pgsql - * using several PostgreSQL tables - * - * This layer stores data read in from the planet.osm file - * and is then read by the backend processing code to - * emit the final geometry-enabled output formats -*/ - -#include "config.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_PTHREAD -#include -#endif - -#include - -#include "osmtypes.h" -#include "output.h" -#include "reprojection.h" -#include "output-pgsql.h" -#include "build_geometry.h" -#include "middle.h" -#include "pgsql.h" -#include "expire-tiles.h" -#include "wildcmp.h" -#include "node-ram-cache.h" - -#define SRID (project_getprojinfo()->srs) - -/* FIXME: Shouldn't malloc this all to begin with but call realloc() - as required. The program will most likely segfault if it reads a - style file with more styles than this */ -#define MAX_STYLES 1000 - -enum table_id { - t_point, t_line, t_poly, t_roads -}; - -static const struct output_options *Options; - -/* enable output of a generated way_area tag to either hstore or its own column */ -static int enable_way_area=1; - -/* Tables to output */ -static struct s_table { - char *name; - const char *type; - PGconn *sql_conn; - char buffer[1024]; - unsigned int buflen; - int copyMode; - char *columns; -} tables [] = { - { .name = "%s_point", .type = "POINT" }, - { .name = "%s_line", .type = "LINESTRING"}, - { .name = "%s_polygon", .type = "GEOMETRY" }, /* Actually POLGYON & MULTIPOLYGON but no way to limit to just these two */ - { .name = "%s_roads", .type = "LINESTRING"} -}; -#define NUM_TABLES ((signed)(sizeof(tables) / sizeof(tables[0]))) - -#define FLAG_POLYGON 1 /* For polygon table */ -#define FLAG_LINEAR 2 /* For lines table */ -#define FLAG_NOCACHE 4 /* Optimisation: don't bother remembering this one */ -#define FLAG_DELETE 8 /* These tags should be simply deleted on sight */ -#define FLAG_PHSTORE 17 /* polygons without own column but listed in hstore this implies FLAG_POLYGON */ -static struct flagsname { - char *name; - int flag; -} tagflags[] = { - { .name = "polygon", .flag = FLAG_POLYGON }, - { .name = "linear", .flag = FLAG_LINEAR }, - { .name = "nocache", .flag = FLAG_NOCACHE }, - { .name = "delete", .flag = FLAG_DELETE }, - { .name = "phstore", .flag = FLAG_PHSTORE } -}; -#define NUM_FLAGS ((signed)(sizeof(tagflags) / sizeof(tagflags[0]))) - -/* Table columns, representing key= tags */ -struct taginfo { - char *name; - char *type; - int flags; - int count; -}; - -static struct taginfo *exportList[4]; /* Indexed by enum table_id */ -static int exportListCount[4]; - -/* Data to generate z-order column and road table - * The name of the roads table is misleading, this table - * is used for any feature to be shown at low zoom. - * This includes railways and administrative boundaries too - */ -static struct { - int offset; - const char *highway; - int roads; -} layers[] = { - { 3, "minor", 0 }, - { 3, "road", 0 }, - { 3, "unclassified", 0 }, - { 3, "residential", 0 }, - { 4, "tertiary_link", 0 }, - { 4, "tertiary", 0 }, - { 6, "secondary_link",1 }, - { 6, "secondary", 1 }, - { 7, "primary_link", 1 }, - { 7, "primary", 1 }, - { 8, "trunk_link", 1 }, - { 8, "trunk", 1 }, - { 9, "motorway_link", 1 }, - { 9, "motorway", 1 } -}; -static const unsigned int nLayers = (sizeof(layers)/sizeof(*layers)); - -static int pgsql_delete_way_from_output(osmid_t osm_id); -static int pgsql_delete_relation_from_output(osmid_t osm_id); -static int pgsql_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int exists); - -void read_style_file( const char *filename ) -{ - FILE *in; - int lineno = 0; - int num_read = 0; - char osmtype[24]; - char tag[64]; - char datatype[24]; - char flags[128]; - int i; - char *str; - int fields; - struct taginfo temp; - char buffer[1024]; - int flag = 0; - - exportList[OSMTYPE_NODE] = malloc( sizeof(struct taginfo) * MAX_STYLES ); - exportList[OSMTYPE_WAY] = malloc( sizeof(struct taginfo) * MAX_STYLES ); - - in = fopen( filename, "rt" ); - if( !in ) - { - fprintf( stderr, "Couldn't open style file '%s': %s\n", filename, strerror(errno) ); - exit_nicely(); - } - - while( fgets( buffer, sizeof(buffer), in) != NULL ) - { - lineno++; - - str = strchr( buffer, '#' ); - if( str ) - *str = '\0'; - - fields = sscanf( buffer, "%23s %63s %23s %127s", osmtype, tag, datatype, flags ); - if( fields <= 0 ) /* Blank line */ - continue; - if( fields < 3 ) - { - fprintf( stderr, "Error reading style file line %d (fields=%d)\n", lineno, fields ); - exit_nicely(); - } - temp.name = strdup(tag); - temp.type = strdup(datatype); - - temp.flags = 0; - for( str = strtok( flags, ",\r\n" ); str; str = strtok(NULL, ",\r\n") ) - { - for( i=0; ienable_hstore)) { - fprintf( stderr, "Error reading style file line %d (fields=%d)\n", lineno, fields ); - fprintf( stderr, "flag 'phstore' is invalid in non-hstore mode\n"); - exit_nicely(); - } - } - if ((temp.flags!=FLAG_DELETE) && ((strchr(temp.name,'?') != NULL) || (strchr(temp.name,'*') != NULL))) { - fprintf( stderr, "wildcard '%s' in non-delete style entry\n",temp.name); - exit_nicely(); - } - - if ((0==strcmp(temp.name,"way_area")) && (temp.flags==FLAG_DELETE)) { - enable_way_area=0; - } - - temp.count = 0; - /* printf("%s %s %d %d\n", temp.name, temp.type, temp.polygon, offset ); */ - - if( strstr( osmtype, "node" ) ) - { - memcpy( &exportList[ OSMTYPE_NODE ][ exportListCount[ OSMTYPE_NODE ] ], &temp, sizeof(temp) ); - exportListCount[ OSMTYPE_NODE ]++; - flag = 1; - } - if( strstr( osmtype, "way" ) ) - { - memcpy( &exportList[ OSMTYPE_WAY ][ exportListCount[ OSMTYPE_WAY ] ], &temp, sizeof(temp) ); - exportListCount[ OSMTYPE_WAY ]++; - flag = 1; - } - if( !flag ) - { - fprintf( stderr, "Weird style line %d\n", lineno ); - exit_nicely(); - } - num_read++; - } - if (ferror(in)) { - perror(filename); - exit_nicely(); - } - if (num_read == 0) { - fprintf(stderr, "Unable to parse any valid columns from the style file. Aborting.\n"); - exit_nicely(); - } - fclose(in); -} - -static void free_style_refs(const char *name, const char *type) -{ - /* Find and remove any other references to these pointers - This would be way easier if we kept a single list of styles - Currently this scales with n^2 number of styles */ - int i,j; - - for (i=0; i sizeof( tables[table].buffer )-10 ) - { - pgsql_CopyData(tables[table].name, sql_conn, buffer); - buflen = 0; - - /* If new data by itself is also too big, output it immediately */ - if( (unsigned)len > sizeof( tables[table].buffer )-10 ) - { - pgsql_CopyData(tables[table].name, sql_conn, sql); - len = 0; - } - } - /* Normal case, just append to buffer */ - if( len > 0 ) - { - strcpy( buffer+buflen, sql ); - buflen += len; - len = 0; - } - - /* If we have completed a line, output it */ - if( buflen > 0 && buffer[buflen-1] == '\n' ) - { - pgsql_CopyData(tables[table].name, sql_conn, buffer); - buflen = 0; - } - - tables[table].buflen = buflen; -} - -static int add_z_order(struct keyval *tags, int *roads) -{ - const char *layer = getItem(tags, "layer"); - const char *highway = getItem(tags, "highway"); - const char *bridge = getItem(tags, "bridge"); - const char *tunnel = getItem(tags, "tunnel"); - const char *railway = getItem(tags, "railway"); - const char *boundary= getItem(tags, "boundary"); - - int z_order = 0; - int l; - unsigned int i; - char z[13]; - - l = layer ? strtol(layer, NULL, 10) : 0; - z_order = 10 * l; - *roads = 0; - - if (highway) { - for (i=0; i - * - * becomes: - * - */ -void compress_tag_name(struct keyval *tags) -{ - const char *name = getItem(tags, "name"); - struct keyval *name_ext = getMatches(tags, "name:"); - struct keyval *p; - char out[2048]; - - if (!name_ext) - return; - - out[0] = '\0'; - if (name) { - strncat(out, name, sizeof(out)-1); - strncat(out, " ", sizeof(out)-1); - } - while((p = popItem(name_ext)) != NULL) { - /* Exclude name:source = "dicataphone" and duplicates */ - if (strcmp(p->key, "name:source") && !strstr(out, p->value)) { - strncat(out, p->value, sizeof(out)-1); - strncat(out, " ", sizeof(out)-1); - } - freeItem(p); - } - free(name_ext); - - /* Remove trailing space */ - out[strlen(out)-1] = '\0'; - /* fprintf(stderr, "*** New name: %s\n", out); */ - updateItem(tags, "name", out); -} - - - -static void pgsql_out_cleanup(void) -{ - int i; - - for (i=0; i tmplen) { - tmpstr=realloc(tmpstr,len); - tmplen=len; - } - strcpy(tmpstr,value); - - if ( !strcmp(type, "int4") ) { - int from, to; - /* For integers we take the first number, or the average if it's a-b */ - items = sscanf(value, "%d-%d", &from, &to); - if ( items == 1 ) { - sprintf(sql, "%d", from); - } else if ( items == 2 ) { - sprintf(sql, "%d", (from + to) / 2); - } else { - sprintf(sql, "\\N"); - } - } else { - /* - try to "repair" real values as follows: - * assume "," to be a decimal mark which need to be replaced by "." - * like int4 take the first number, or the average if it's a-b - * assume SI unit (meters) - * convert feet to meters (1 foot = 0.3048 meters) - * reject anything else - */ - if ( !strcmp(type, "real") ) { - int i,slen; - float from,to; - - slen=strlen(value); - for (i=0;inext->key != NULL) - { - - /* hard exclude z_order tag and keys which have their own column */ - if ((xtags->next->has_column) || (strcmp("z_order",xtags->next->key)==0)) { - /* update the tag-pointer to point to the next tag */ - xtags = xtags->next; - continue; - } - - /* - hstore ASCII representation looks like - ""=>"" - - we need at least strlen(key)+strlen(value)+6+'\0' bytes - in theory any single character could also be escaped - thus we need an additional factor of 2. - The maximum lenght of a single hstore element is thus - calcuated as follows: - */ - hlen=2 * (strlen(xtags->next->key) + strlen(xtags->next->value)) + 7; - - /* if the sql buffer is too small */ - if (hlen > sqllen) { - sqllen = hlen; - sql = realloc(sql, sqllen); - } - - /* pack the tag with its value into the hstore */ - keyval2hstore(sql, xtags->next); - copy_to_table(table, sql); - - /* update the tag-pointer to point to the next tag */ - xtags = xtags->next; - - /* if the tag has a follow up, add a comma to the end */ - if (xtags->next->key != NULL) - copy_to_table(table, ","); - } - - /* finish the hstore column by placing a TAB into the data stream */ - copy_to_table(table, "\t"); - - /* the main hstore-column has now been written */ -} - -/* write an hstore column to the database */ -static void write_hstore_columns(enum table_id table, struct keyval *tags) -{ - static char *sql; - static size_t sqllen=0; - char *shortkey; - /* the index of the current hstore column */ - int i_hstore_column; - int found; - struct keyval *xtags; - char *pos; - size_t hlen; - - /* sql buffer */ - if (sqllen==0) { - sqllen=2048; - sql=malloc(sqllen); - } - - /* iterate over all configured hstore colums in the options */ - for(i_hstore_column = 0; i_hstore_column < Options->n_hstore_columns; i_hstore_column++) - { - /* did this node have a tag that matched the current hstore column */ - found = 0; - - /* a clone of the tags pointer */ - xtags = tags; - - /* while this tags has a follow-up.. */ - while (xtags->next->key != NULL) { - - /* check if the tag's key starts with the name of the hstore column */ - pos = strstr(xtags->next->key, Options->hstore_columns[i_hstore_column]); - - /* and if it does.. */ - if(pos == xtags->next->key) - { - /* remember we found one */ - found=1; - - /* generate the short key name */ - shortkey = xtags->next->key + strlen(Options->hstore_columns[i_hstore_column]); - - /* calculate the size needed for this hstore entry */ - hlen=2*(strlen(shortkey)+strlen(xtags->next->value))+7; - - /* if the sql buffer is too small */ - if (hlen > sqllen) { - /* resize it */ - sqllen=hlen; - sql=realloc(sql,sqllen); - } - - /* and pack the shortkey with its value into the hstore */ - keyval2hstore_manual(sql, shortkey, xtags->next->value); - copy_to_table(table, sql); - - /* update the tag-pointer to point to the next tag */ - xtags=xtags->next; - - /* if the tag has a follow up, add a comma to the end */ - if (xtags->next->key != NULL) - copy_to_table(table, ","); - } - else - { - /* update the tag-pointer to point to the next tag */ - xtags=xtags->next; - } - } - - /* if no matching tag has been found, write a NULL */ - if(!found) - copy_to_table(table, "\\N"); - - /* finish the hstore column by placing a TAB into the data stream */ - copy_to_table(table, "\t"); - } - - /* all hstore-columns have now been written */ -} - - -/* example from: pg_dump -F p -t planet_osm gis -COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, building, bridge, layer, way) FROM stdin; -17959841 \N \N \N \N \N \N \N bus_stop \N \N \N \N \N \N -\N 0101000020E610000030CCA462B6C3D4BF92998C9B38E04940 -17401934 The Horn \N \N \N \N \N \N \N \N pub \N \N \N \N -\N 0101000020E6100000C12FC937140FD5BFB4D2F4FB0CE04940 -... - -mine - 01 01000000 48424298424242424242424256427364 -psql - 01 01000020 E6100000 30CCA462B6C3D4BF92998C9B38E04940 - 01 01000020 E6100000 48424298424242424242424256427364 -0x2000_0000 = hasSRID, following 4 bytes = srid, not supported by geos WKBWriter -Workaround - output SRID=4326; -*/ - -static int pgsql_out_node(osmid_t id, struct keyval *tags, double node_lat, double node_lon) -{ - - static char *sql; - static size_t sqllen=0; - int i; - struct keyval *tag; - - if (sqllen==0) { - sqllen=2048; - sql=malloc(sqllen); - } - - expire_tiles_from_bbox(node_lon, node_lat, node_lon, node_lat); - sprintf(sql, "%" PRIdOSMID "\t", id); - copy_to_table(t_point, sql); - - for (i=0; i < exportListCount[OSMTYPE_NODE]; i++) { - if( exportList[OSMTYPE_NODE][i].flags & FLAG_DELETE ) - continue; - if( (exportList[OSMTYPE_NODE][i].flags & FLAG_PHSTORE) == FLAG_PHSTORE) - continue; - if ((tag = getTag(tags, exportList[OSMTYPE_NODE][i].name))) - { - escape_type(sql, sqllen, tag->value, exportList[OSMTYPE_NODE][i].type); - exportList[OSMTYPE_NODE][i].count++; - if (HSTORE_NORM==Options->enable_hstore) - tag->has_column=1; - } - else - sprintf(sql, "\\N"); - - copy_to_table(t_point, sql); - copy_to_table(t_point, "\t"); - } - - /* hstore columns */ - write_hstore_columns(t_point, tags); - - /* check if a regular hstore is requested */ - if (Options->enable_hstore) - write_hstore(t_point, tags); - -#ifdef FIXED_POINT - // guarantee that we use the same values as in the node cache - scale = Options->scale; - node_lon = FIX_TO_DOUBLE(DOUBLE_TO_FIX(node_lon)); - node_lat = FIX_TO_DOUBLE(DOUBLE_TO_FIX(node_lat)); -#endif - - sprintf(sql, "SRID=%d;POINT(%.15g %.15g)", SRID, node_lon, node_lat); - copy_to_table(t_point, sql); - copy_to_table(t_point, "\n"); - - return 0; -} - - - -static void write_wkts(osmid_t id, struct keyval *tags, const char *wkt, enum table_id table) -{ - - static char *sql; - static size_t sqllen=0; - int j; - struct keyval *tag; - - if (sqllen==0) { - sqllen=2048; - sql=malloc(sqllen); - } - - sprintf(sql, "%" PRIdOSMID "\t", id); - copy_to_table(table, sql); - - for (j=0; j < exportListCount[OSMTYPE_WAY]; j++) { - if( exportList[OSMTYPE_WAY][j].flags & FLAG_DELETE ) - continue; - if( (exportList[OSMTYPE_WAY][j].flags & FLAG_PHSTORE) == FLAG_PHSTORE) - continue; - if ((tag = getTag(tags, exportList[OSMTYPE_WAY][j].name))) - { - exportList[OSMTYPE_WAY][j].count++; - escape_type(sql, sqllen, tag->value, exportList[OSMTYPE_WAY][j].type); - if (HSTORE_NORM==Options->enable_hstore) - tag->has_column=1; - } - else - sprintf(sql, "\\N"); - - copy_to_table(table, sql); - copy_to_table(table, "\t"); - } - - /* hstore columns */ - write_hstore_columns(table, tags); - - /* check if a regular hstore is requested */ - if (Options->enable_hstore) - write_hstore(table, tags); - - sprintf(sql, "SRID=%d;", SRID); - copy_to_table(table, sql); - copy_to_table(table, wkt); - copy_to_table(table, "\n"); -} - -static int tag_indicates_polygon(enum OsmType type, const char *key) -{ - int i; - - if (!strcmp(key, "area")) - return 1; - - for (i=0; i < exportListCount[type]; i++) { - if( strcmp( exportList[type][i].name, key ) == 0 ) - return exportList[type][i].flags & FLAG_POLYGON; - } - - return 0; -} - -/* Go through the given tags and determine the union of flags. Also remove - * any tags from the list that we don't know about */ -unsigned int pgsql_filter_tags(enum OsmType type, struct keyval *tags, int *polygon) -{ - int i, filter = 1; - int flags = 0; - int add_area_tag = 0; - - const char *area; - struct keyval *item; - struct keyval temp; - initList(&temp); - - /* We used to only go far enough to determine if it's a polygon or not, but now we go through and filter stuff we don't need */ - while( (item = popItem(tags)) != NULL ) - { - /* Allow named islands to appear as polygons */ - if (!strcmp("natural",item->key) && !strcmp("coastline",item->value)) - { - add_area_tag = 1; - } - - /* Discard natural=coastline tags (we render these from a shapefile instead) */ - if (!Options->keep_coastlines && !strcmp("natural",item->key) && !strcmp("coastline",item->value)) - { - freeItem( item ); - item = NULL; - continue; - } - - for (i=0; i < exportListCount[type]; i++) - { - if (wildMatch( exportList[type][i].name, item->key )) - { - if( exportList[type][i].flags & FLAG_DELETE ) - { - freeItem( item ); - item = NULL; - break; - } - - filter = 0; - flags |= exportList[type][i].flags; - - pushItem( &temp, item ); - item = NULL; - break; - } - } - - /** if tag not found in list of exports: */ - if (i == exportListCount[type]) - { - if (Options->enable_hstore) { - /* with hstore, copy all tags... */ - pushItem(&temp, item); - /* ... but if hstore_match_only is set then don't take this - as a reason for keeping the object */ - if ( - !Options->hstore_match_only - && strcmp("osm_uid",item->key) - && strcmp("osm_user",item->key) - && strcmp("osm_timestamp",item->key) - && strcmp("osm_version",item->key) - && strcmp("osm_changeset",item->key) - ) filter = 0; - } else if (Options->n_hstore_columns) { - /* does this column match any of the hstore column prefixes? */ - int j; - for (j = 0; j < Options->n_hstore_columns; j++) { - char *pos = strstr(item->key, Options->hstore_columns[j]); - if (pos == item->key) { - pushItem(&temp, item); - /* ... but if hstore_match_only is set then don't take this - as a reason for keeping the object */ - if ( - !Options->hstore_match_only - && strcmp("osm_uid",item->key) - && strcmp("osm_user",item->key) - && strcmp("osm_timestamp",item->key) - && strcmp("osm_version",item->key) - && strcmp("osm_changeset",item->key) - ) filter = 0; - break; - } - } - /* if not, skip the tag */ - if (j == Options->n_hstore_columns) { - freeItem(item); - } - } else { - freeItem(item); - } - item = NULL; - } - } - - /* Move from temp list back to original list */ - while( (item = popItem(&temp)) != NULL ) - pushItem( tags, item ); - - *polygon = flags & FLAG_POLYGON; - - /* Special case allowing area= to override anything else */ - if ((area = getItem(tags, "area"))) { - if (!strcmp(area, "yes") || !strcmp(area, "true") ||!strcmp(area, "1")) - *polygon = 1; - else if (!strcmp(area, "no") || !strcmp(area, "false") || !strcmp(area, "0")) - *polygon = 0; - } else { - /* If we need to force this as a polygon, append an area tag */ - if (add_area_tag) { - addItem(tags, "area", "yes", 0); - *polygon = 1; - } - } - - return filter; -} - -/* -COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, bu -ilding, bridge, layer, way) FROM stdin; -198497 Bedford Road \N \N \N \N \N \N residential \N \N \N \N \N \N \N 0102000020E610000004000000452BF702B342D5BF1C60E63BF8DF49406B9C4D470037D5BF5471E316F3DF4940DFA815A6EF35D5BF9AE95E27F5DF4940B41EB -E4C1421D5BF24D06053E7DF4940 -212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D -5BF5BB39597FCDF4940 -*/ -static int pgsql_out_way(osmid_t id, struct keyval *tags, struct osmNode *nodes, int count, int exists) -{ - int polygon = 0, roads = 0; - int i, wkt_size; - double split_at; - double area; - - /* If the flag says this object may exist already, delete it first */ - if(exists) { - pgsql_delete_way_from_output(id); - Options->mid->way_changed(id); - } - - if (pgsql_filter_tags(OSMTYPE_WAY, tags, &polygon) || add_z_order(tags, &roads)) - return 0; - - /* Split long ways after around 1 degree or 100km */ - if (Options->projection == PROJ_LATLONG) - split_at = 1; - else - split_at = 100 * 1000; - - wkt_size = get_wkt_split(nodes, count, polygon, split_at); - - for (i=0;i 0.0) && enable_way_area) { - char tmp[32]; - snprintf(tmp, sizeof(tmp), "%g", area); - addItem(tags, "way_area", tmp, 0); - } - write_wkts(id, tags, wkt, t_poly); - } else { - expire_tiles_from_nodes_line(nodes, count); - write_wkts(id, tags, wkt, t_line); - if (roads) - write_wkts(id, tags, wkt, t_roads); - } - } - free(wkt); - } - clear_wkts(); - - return 0; -} - -static int pgsql_out_relation(osmid_t id, struct keyval *rel_tags, struct osmNode **xnodes, struct keyval *xtags, int *xcount, osmid_t *xid, const char **xrole) -{ - int i, wkt_size; - int polygon = 0, roads = 0; - int make_polygon = 0; - int make_boundary = 0; - struct keyval tags, *p, poly_tags; - char *type; - double split_at; - -#if 0 - fprintf(stderr, "Got relation with counts:"); - for (i=0; xcount[i]; i++) - fprintf(stderr, " %d", xcount[i]); - fprintf(stderr, "\n"); -#endif - /* Get the type, if there's no type we don't care */ - type = getItem(rel_tags, "type"); - if( !type ) - return 0; - - initList(&tags); - initList(&poly_tags); - - /* Clone tags from relation */ - p = rel_tags->next; - while (p != rel_tags) { - /* For routes, we convert name to route_name */ - if ((strcmp(type, "route") == 0) && (strcmp(p->key, "name") ==0)) - addItem(&tags, "route_name", p->value, 1); - else if (strcmp(p->key, "type")) /* drop type= */ - addItem(&tags, p->key, p->value, 1); - p = p->next; - } - - if( strcmp(type, "route") == 0 ) - { - const char *state = getItem(rel_tags, "state"); - const char *netw = getItem(rel_tags, "network"); - int networknr = -1; - - if (state == NULL) { - state = ""; - } - - if (netw != NULL) { - if (strcmp(netw, "lcn") == 0) { - networknr = 10; - if (strcmp(state, "alternate") == 0) { - addItem(&tags, "lcn", "alternate", 1); - } else if (strcmp(state, "connection") == 0) { - addItem(&tags, "lcn", "connection", 1); - } else { - addItem(&tags, "lcn", "yes", 1); - } - } else if (strcmp(netw, "rcn") == 0) { - networknr = 11; - if (strcmp(state, "alternate") == 0) { - addItem(&tags, "rcn", "alternate", 1); - } else if (strcmp(state, "connection") == 0) { - addItem(&tags, "rcn", "connection", 1); - } else { - addItem(&tags, "rcn", "yes", 1); - } - } else if (strcmp(netw, "ncn") == 0) { - networknr = 12; - if (strcmp(state, "alternate") == 0) { - addItem(&tags, "ncn", "alternate", 1); - } else if (strcmp(state, "connection") == 0) { - addItem(&tags, "ncn", "connection", 1); - } else { - addItem(&tags, "ncn", "yes", 1); - } - - - } else if (strcmp(netw, "lwn") == 0) { - networknr = 20; - if (strcmp(state, "alternate") == 0) { - addItem(&tags, "lwn", "alternate", 1); - } else if (strcmp(state, "connection") == 0) { - addItem(&tags, "lwn", "connection", 1); - } else { - addItem(&tags, "lwn", "yes", 1); - } - } else if (strcmp(netw, "rwn") == 0) { - networknr = 21; - if (strcmp(state, "alternate") == 0) { - addItem(&tags, "rwn", "alternate", 1); - } else if (strcmp(state, "connection") == 0) { - addItem(&tags, "rwn", "connection", 1); - } else { - addItem(&tags, "rwn", "yes", 1); - } - } else if (strcmp(netw, "nwn") == 0) { - networknr = 22; - if (strcmp(state, "alternate") == 0) { - addItem(&tags, "nwn", "alternate", 1); - } else if (strcmp(state, "connection") == 0) { - addItem(&tags, "nwn", "connection", 1); - } else { - addItem(&tags, "nwn", "yes", 1); - } - } - } - - if (getItem(rel_tags, "preferred_color") != NULL) { - const char *a = getItem(rel_tags, "preferred_color"); - if (strcmp(a, "0") == 0 || strcmp(a, "1") == 0 || strcmp(a, "2") == 0 || strcmp(a, "3") == 0 || strcmp(a, "4") == 0) { - addItem(&tags, "route_pref_color", a, 1); - } else { - addItem(&tags, "route_pref_color", "0", 1); - } - } else { - addItem(&tags, "route_pref_color", "0", 1); - } - - if (getItem(rel_tags, "ref") != NULL) { - if (networknr == 10) { - addItem(&tags, "lcn_ref", getItem(rel_tags, "ref"), 1); - } else if (networknr == 11) { - addItem(&tags, "rcn_ref", getItem(rel_tags, "ref"), 1); - } else if (networknr == 12) { - addItem(&tags, "ncn_ref", getItem(rel_tags, "ref"), 1); - } else if (networknr == 20) { - addItem(&tags, "lwn_ref", getItem(rel_tags, "ref"), 1); - } else if (networknr == 21) { - addItem(&tags, "rwn_ref", getItem(rel_tags, "ref"), 1); - } else if (networknr == 22) { - addItem(&tags, "nwn_ref", getItem(rel_tags, "ref"), 1); - } - } - } - else if( strcmp( type, "boundary" ) == 0 ) - { - /* Boundaries will get converted into multiple geometries: - - Linear features will end up in the line and roads tables (useful for admin boundaries) - - Polygon features also go into the polygon table (useful for national_forests) - The edges of the polygon also get treated as linear fetaures allowing these to be rendered seperately. */ - make_boundary = 1; - } - else if( strcmp( type, "multipolygon" ) == 0 && getItem(&tags, "boundary") ) - { - /* Treat type=multipolygon exactly like type=boundary if it has a boundary tag. */ - make_boundary = 1; - } - else if( strcmp( type, "multipolygon" ) == 0 ) - { - make_polygon = 1; - - /* Copy the tags from the outer way(s) if the relation is untagged */ - /* or if there is just a name tag, people seem to like naming relations */ - if (!listHasData(&tags) || ((countList(&tags)==1) && getItem(&tags, "name"))) { - for (i=0; xcount[i]; i++) { - if (xrole[i] && !strcmp(xrole[i], "inner")) - continue; - - p = xtags[i].next; - while (p != &(xtags[i])) { - addItem(&tags, p->key, p->value, 1); - p = p->next; - } - } - } - - /* Collect a list of polygon-like tags, these are used later to - identify if an inner rings looks like it should be rendered seperately */ - p = tags.next; - while (p != &tags) { - if (tag_indicates_polygon(OSMTYPE_WAY, p->key)) { - addItem(&poly_tags, p->key, p->value, 1); - } - p = p->next; - } - } - else - { - /* Unknown type, just exit */ - resetList(&tags); - resetList(&poly_tags); - return 0; - } - - if (pgsql_filter_tags(OSMTYPE_WAY, &tags, &polygon) || add_z_order(&tags, &roads)) { - resetList(&tags); - resetList(&poly_tags); - return 0; - } - - /* Split long linear ways after around 1 degree or 100km (polygons not effected) */ - if (Options->projection == PROJ_LATLONG) - split_at = 1; - else - split_at = 100 * 1000; - - wkt_size = build_geometry(id, xnodes, xcount, make_polygon, Options->enable_multi, split_at); - - if (!wkt_size) { - resetList(&tags); - resetList(&poly_tags); - return 0; - } - - for (i=0;i 0.0) && enable_way_area) { - char tmp[32]; - snprintf(tmp, sizeof(tmp), "%g", area); - addItem(&tags, "way_area", tmp, 0); - } - write_wkts(-id, &tags, wkt, t_poly); - } else { - write_wkts(-id, &tags, wkt, t_line); - if (roads) - write_wkts(-id, &tags, wkt, t_roads); - } - } - free(wkt); - } - - clear_wkts(); - - /* If we are creating a multipolygon then we - mark each member so that we can skip them during iterate_ways - but only if the polygon-tags look the same as the outer ring */ - if (make_polygon) { - for (i=0; xcount[i]; i++) { - int match = 0; - struct keyval *p = poly_tags.next; - while (p != &poly_tags) { - const char *v = getItem(&xtags[i], p->key); - if (!v || strcmp(v, p->value)) { - match = 0; - break; - } - match = 1; - p = p->next; - } - if (match) { - Options->mid->ways_done(xid[i]); - pgsql_delete_way_from_output(xid[i]); - } - } - } - - /* If we are making a boundary then also try adding any relations which form complete rings - The linear variants will have already been processed above */ - if (make_boundary) { - wkt_size = build_geometry(id, xnodes, xcount, 1, Options->enable_multi, split_at); - for (i=0;i 0.0) && enable_way_area) { - char tmp[32]; - snprintf(tmp, sizeof(tmp), "%g", area); - addItem(&tags, "way_area", tmp, 0); - } - write_wkts(-id, &tags, wkt, t_poly); - } - } - free(wkt); - } - clear_wkts(); - } - - resetList(&tags); - resetList(&poly_tags); - return 0; -} - -static int pgsql_out_connect(const struct output_options *options, int startTransaction) { - int i; - for (i=0; iconninfo); - - /* Check to see that the backend connection was successfully made */ - if (PQstatus(sql_conn) != CONNECTION_OK) { - fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn)); - return 1; - } - tables[i].sql_conn = sql_conn; - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;"); - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %s WHERE osm_id = $1;\n", tables[i].name); - if (startTransaction) - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "BEGIN"); - } - return 0; -} - -static int pgsql_out_start(const struct output_options *options) -{ - char *sql, tmp[256]; - PGresult *res; - int i,j; - unsigned int sql_len; - int their_srid; - int i_hstore_column; - enum OsmType type; - int numTags; - struct taginfo *exportTags; - - Options = options; - - read_style_file( options->style ); - - sql_len = 2048; - sql = malloc(sql_len); - assert(sql); - - for (i=0; iprefix) + strlen(tables[i].name) + 1 ); - sprintf( temp, tables[i].name, options->prefix ); - tables[i].name = temp; - } - fprintf(stderr, "Setting up table: %s\n", tables[i].name); - sql_conn = PQconnectdb(options->conninfo); - - /* Check to see that the backend connection was successfully made */ - if (PQstatus(sql_conn) != CONNECTION_OK) { - fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(sql_conn)); - exit_nicely(); - } - tables[i].sql_conn = sql_conn; - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;"); - - if (!options->append) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s", tables[i].name); - } - else - { - sprintf(sql, "SELECT srid FROM geometry_columns WHERE f_table_name='%s';", tables[i].name); - res = PQexec(sql_conn, sql); - if (!((PQntuples(res) == 1) && (PQnfields(res) == 1))) - { - fprintf(stderr, "Problem reading geometry information for table %s - does it exist?\n", tables[i].name); - exit_nicely(); - } - their_srid = atoi(PQgetvalue(res, 0, 0)); - PQclear(res); - if (their_srid != SRID) - { - fprintf(stderr, "SRID mismatch: cannot append to table %s (SRID %d) using selected SRID %d\n", tables[i].name, their_srid, SRID); - exit_nicely(); - } - } - - /* These _tmp tables can be left behind if we run out of disk space */ - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE IF EXISTS %s_tmp", tables[i].name); - - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "BEGIN"); - - type = (i == t_point)?OSMTYPE_NODE:OSMTYPE_WAY; - numTags = exportListCount[type]; - exportTags = exportList[type]; - if (!options->append) { - sprintf(sql, "CREATE TABLE %s ( osm_id " POSTGRES_OSMID_TYPE, tables[i].name ); - for (j=0; j < numTags; j++) { - if( exportTags[j].flags & FLAG_DELETE ) - continue; - if( (exportTags[j].flags & FLAG_PHSTORE ) == FLAG_PHSTORE) - continue; - sprintf(tmp, ",\"%s\" %s", exportTags[j].name, exportTags[j].type); - if (strlen(sql) + strlen(tmp) + 1 > sql_len) { - sql_len *= 2; - sql = realloc(sql, sql_len); - assert(sql); - } - strcat(sql, tmp); - } - for(i_hstore_column = 0; i_hstore_column < Options->n_hstore_columns; i_hstore_column++) - { - strcat(sql, ",\""); - strcat(sql, Options->hstore_columns[i_hstore_column]); - strcat(sql, "\" hstore "); - } - if (Options->enable_hstore) { - strcat(sql, ",tags hstore"); - } - strcat(sql, ")"); - if (Options->tblsmain_data) { - sprintf(sql + strlen(sql), " TABLESPACE %s", Options->tblsmain_data); - } - strcat(sql, "\n"); - - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", sql); - pgsql_exec(sql_conn, PGRES_TUPLES_OK, "SELECT AddGeometryColumn('%s', 'way', %d, '%s', 2 );\n", - tables[i].name, SRID, tables[i].type ); - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ALTER TABLE %s ALTER COLUMN way SET NOT NULL;\n", tables[i].name); - /* slim mode needs this to be able to apply diffs */ - if (Options->slim && !Options->droptemp) { - sprintf(sql, "CREATE INDEX %s_pkey ON %s USING BTREE (osm_id)", tables[i].name, tables[i].name); - if (Options->tblsmain_index) { - sprintf(sql + strlen(sql), " TABLESPACE %s\n", Options->tblsmain_index); - } - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "%s", sql); - } - } else { - /* Add any new columns referenced in the default.style */ - PGresult *res; - sprintf(sql, "SELECT * FROM %s LIMIT 0;\n", tables[i].name); - res = PQexec(sql_conn, sql); - if (PQresultStatus(res) != PGRES_TUPLES_OK) { - fprintf(stderr, "Error, failed to query table %s\n%s\n", tables[i].name, sql); - exit_nicely(); - } - for (j=0; j < numTags; j++) { - if( exportTags[j].flags & FLAG_DELETE ) - continue; - if( (exportTags[j].flags & FLAG_PHSTORE) == FLAG_PHSTORE) - continue; - sprintf(tmp, "\"%s\"", exportTags[j].name); - if (PQfnumber(res, tmp) < 0) { -#if 0 - fprintf(stderr, "Append failed. Column \"%s\" is missing from \"%s\"\n", exportTags[j].name, tables[i].name); - exit_nicely(); -#else - fprintf(stderr, "Adding new column \"%s\" to \"%s\"\n", exportTags[j].name, tables[i].name); - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ALTER TABLE %s ADD COLUMN \"%s\" %s;\n", tables[i].name, exportTags[j].name, exportTags[j].type); -#endif - } - /* Note: we do not verify the type or delete unused columns */ - } - - PQclear(res); - - /* change the type of the geometry column if needed - this can only change to a more permisive type */ - } - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %s WHERE osm_id = $1;\n", tables[i].name); - - /* Generate column list for COPY */ - strcpy(sql, "osm_id"); - for (j=0; j < numTags; j++) { - if( exportTags[j].flags & FLAG_DELETE ) - continue; - if( (exportTags[j].flags & FLAG_PHSTORE ) == FLAG_PHSTORE) - continue; - sprintf(tmp, ",\"%s\"", exportTags[j].name); - - if (strlen(sql) + strlen(tmp) + 1 > sql_len) { - sql_len *= 2; - sql = realloc(sql, sql_len); - assert(sql); - } - strcat(sql, tmp); - } - - for(i_hstore_column = 0; i_hstore_column < Options->n_hstore_columns; i_hstore_column++) - { - strcat(sql, ",\""); - strcat(sql, Options->hstore_columns[i_hstore_column]); - strcat(sql, "\" "); - } - - if (Options->enable_hstore) strcat(sql,",tags"); - - tables[i].columns = strdup(sql); - pgsql_exec(sql_conn, PGRES_COPY_IN, "COPY %s (%s,way) FROM STDIN", tables[i].name, tables[i].columns); - - tables[i].copyMode = 1; - } - free(sql); - - expire_tiles_init(options); - - options->mid->start(options); - - return 0; -} - -static void pgsql_pause_copy(struct s_table *table) -{ - PGresult *res; - int stop; - - if( !table->copyMode ) - return; - - /* Terminate any pending COPY */ - stop = PQputCopyEnd(table->sql_conn, NULL); - if (stop != 1) { - fprintf(stderr, "COPY_END for %s failed: %s\n", table->name, PQerrorMessage(table->sql_conn)); - exit_nicely(); - } - - res = PQgetResult(table->sql_conn); - if (PQresultStatus(res) != PGRES_COMMAND_OK) { - fprintf(stderr, "COPY_END for %s failed: %s\n", table->name, PQerrorMessage(table->sql_conn)); - PQclear(res); - exit_nicely(); - } - PQclear(res); - table->copyMode = 0; -} - -static void pgsql_out_close(int stopTransaction) { - int i; - for (i=0; isql_conn; - - if( table->buflen != 0 ) - { - fprintf( stderr, "Internal error: Buffer for %s has %d bytes after end copy", table->name, table->buflen ); - exit_nicely(); - } - - pgsql_pause_copy(table); - if (!Options->append) - { - time_t start, end; - time(&start); - fprintf(stderr, "Sorting data and creating indexes for %s\n", table->name); - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ANALYZE %s;\n", table->name); - fprintf(stderr, "Analyzing %s finished\n", table->name); - if (Options->tblsmain_data) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE TABLE %s_tmp " - "TABLESPACE %s AS SELECT * FROM %s ORDER BY way;\n", - table->name, Options->tblsmain_data, table->name); - } else { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE TABLE %s_tmp AS SELECT * FROM %s ORDER BY way;\n", table->name, table->name); - } - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "DROP TABLE %s;\n", table->name); - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ALTER TABLE %s_tmp RENAME TO %s;\n", table->name, table->name); - fprintf(stderr, "Copying %s to cluster by geometry finished\n", table->name); - fprintf(stderr, "Creating geometry index on %s\n", table->name); - if (Options->tblsmain_index) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_index ON %s USING GIST (way) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index); - } else { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_index ON %s USING GIST (way);\n", table->name, table->name); - } - - /* slim mode needs this to be able to apply diffs */ - if (Options->slim && !Options->droptemp) - { - fprintf(stderr, "Creating osm_id index on %s\n", table->name); - if (Options->tblsmain_index) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_pkey ON %s USING BTREE (osm_id) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index); - } else { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_pkey ON %s USING BTREE (osm_id);\n", table->name, table->name); - } - } - /* Create hstore index if selected */ - if (Options->enable_hstore_index) { - fprintf(stderr, "Creating hstore indexes on %s\n", table->name); - if (Options->tblsmain_index) { - if (HSTORE_NONE != (Options->enable_hstore)) - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_tags_index ON %s USING GIN (tags) TABLESPACE %s;\n", table->name, table->name, Options->tblsmain_index); - for(i_column = 0; i_column < Options->n_hstore_columns; i_column++) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_hstore_%i_index ON %s USING GIN (\"%s\") TABLESPACE %s;\n", - table->name, i_column,table->name, Options->hstore_columns[i_column], Options->tblsmain_index); - } - } else { - if (HSTORE_NONE != (Options->enable_hstore)) - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_tags_index ON %s USING GIN (tags);\n", table->name, table->name); - for(i_column = 0; i_column < Options->n_hstore_columns; i_column++) { - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "CREATE INDEX %s_hstore_%i_index ON %s USING GIN (\"%s\");\n", table->name, i_column,table->name, Options->hstore_columns[i_column]); - } - } - } - fprintf(stderr, "Creating indexes on %s finished\n", table->name); - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "GRANT SELECT ON %s TO PUBLIC;\n", table->name); - pgsql_exec(sql_conn, PGRES_COMMAND_OK, "ANALYZE %s;\n", table->name); - time(&end); - fprintf(stderr, "All indexes on %s created in %ds\n", table->name, (int)(end - start)); - } - PQfinish(sql_conn); - table->sql_conn = NULL; - - fprintf(stderr, "Completed %s\n", table->name); - free(table->name); - free(table->columns); - return NULL; -} - -static void pgsql_out_stop() -{ - int i; -#ifdef HAVE_PTHREAD - pthread_t threads[NUM_TABLES]; -#endif - - /* Commit the transactions, so that multiple processes can - * access the data simultanious to process the rest in parallel - * as well as see the newly created tables. - */ - pgsql_out_commit(); - Options->mid->commit(); - /* To prevent deadlocks in parallel processing, the mid tables need - * to stay out of a transaction. In this stage output tables are only - * written to and not read, so they can be processed as several parallel - * independent transactions - */ - for (i=0; imid->iterate_ways( pgsql_out_way ); - pgsql_out_commit(); - Options->mid->commit(); - - /* Processing any remaing to be processed relations */ - /* During this stage output tables also need to stay out of - * extended transactions, as the delete_way_from_output, called - * from process_relation, can deadlock if using multi-processing. - */ - Options->mid->iterate_relations( pgsql_process_relation ); - -#ifdef HAVE_PTHREAD - if (Options->parallel_indexing) { - for (i=0; imid->stop(); - - for (i=0; imid->stop(); - for (i=0; imid->nodes_set(id, lat, lon, tags); - if( !filter ) - pgsql_out_node(id, tags, lat, lon); - return 0; -} - -static int pgsql_add_way(osmid_t id, osmid_t *nds, int nd_count, struct keyval *tags) -{ - int polygon = 0; - - /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */ - int filter = pgsql_filter_tags(OSMTYPE_WAY, tags, &polygon); - - /* If this isn't a polygon then it can not be part of a multipolygon - Hence only polygons are "pending" */ - Options->mid->ways_set(id, nds, nd_count, tags, (!filter && polygon) ? 1 : 0); - - if( !polygon && !filter ) - { - /* Get actual node data and generate output */ - struct osmNode *nodes = malloc( sizeof(struct osmNode) * nd_count ); - int count = Options->mid->nodes_get_list( nodes, nds, nd_count ); - pgsql_out_way(id, tags, nodes, count, 0); - free(nodes); - } - return 0; -} - -/* This is the workhorse of pgsql_add_relation, split out because it is used as the callback for iterate relations */ -static int pgsql_process_relation(osmid_t id, struct member *members, int member_count, struct keyval *tags, int exists) -{ - int i, j, count, count2; - osmid_t *xid2 = malloc( (member_count+1) * sizeof(osmid_t) ); - osmid_t *xid; - const char **xrole = malloc( (member_count+1) * sizeof(const char *) ); - int *xcount = malloc( (member_count+1) * sizeof(int) ); - struct keyval *xtags = malloc( (member_count+1) * sizeof(struct keyval) ); - struct osmNode **xnodes = malloc( (member_count+1) * sizeof(struct osmNode*) ); - - /* If the flag says this object may exist already, delete it first */ - if(exists) - pgsql_delete_relation_from_output(id); - - count = 0; - for( i=0; imid->ways_get_list(xid2, count, &xid, xtags, xnodes, xcount); - - for (i = 0; i < count2; i++) { - for (j = i; j < member_count; j++) { - if (members[j].id == xid[i]) break; - } - xrole[i] = members[j].role; - } - xnodes[count2] = NULL; - xcount[count2] = 0; - xid[count2] = 0; - xrole[count2] = NULL; - - /* At some point we might want to consider storing the retreived data in the members, rather than as seperate arrays */ - pgsql_out_relation(id, tags, xnodes, xtags, xcount, xid, xrole); - - for( i=0; imid->relations_set) - Options->mid->relations_set(id, members, member_count, tags); - - /* Only a limited subset of type= is supported, ignore other */ - if ( (strcmp(type, "route") != 0) && (strcmp(type, "multipolygon") != 0) && (strcmp(type, "boundary") != 0)) - return 0; - - - return pgsql_process_relation(id, members, member_count, tags, 0); -} -#define UNUSED __attribute__ ((unused)) - -/* Delete is easy, just remove all traces of this object. We don't need to - * worry about finding objects that depend on it, since the same diff must - * contain the change for that also. */ -static int pgsql_delete_node(osmid_t osm_id) -{ - if( !Options->slim ) - { - fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); - exit_nicely(); - } - pgsql_pause_copy(&tables[t_point]); - if ( expire_tiles_from_db(tables[t_point].sql_conn, osm_id) != 0) - pgsql_exec(tables[t_point].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_point].name, osm_id ); - - Options->mid->nodes_delete(osm_id); - return 0; -} - -/* Seperated out because we use it elsewhere */ -static int pgsql_delete_way_from_output(osmid_t osm_id) -{ - /* Optimisation: we only need this is slim mode */ - if( !Options->slim ) - return 0; - /* in droptemp mode we don't have indices and this takes ages. */ - if (Options->droptemp) - return 0; - pgsql_pause_copy(&tables[t_roads]); - pgsql_pause_copy(&tables[t_line]); - pgsql_pause_copy(&tables[t_poly]); - pgsql_exec(tables[t_roads].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_roads].name, osm_id ); - if ( expire_tiles_from_db(tables[t_line].sql_conn, osm_id) != 0) - pgsql_exec(tables[t_line].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_line].name, osm_id ); - if ( expire_tiles_from_db(tables[t_poly].sql_conn, osm_id) != 0) - pgsql_exec(tables[t_poly].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_poly].name, osm_id ); - return 0; -} - -static int pgsql_delete_way(osmid_t osm_id) -{ - if( !Options->slim ) - { - fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); - exit_nicely(); - } - pgsql_delete_way_from_output(osm_id); - Options->mid->ways_delete(osm_id); - return 0; -} - -/* Relations are identified by using negative IDs */ -static int pgsql_delete_relation_from_output(osmid_t osm_id) -{ - pgsql_pause_copy(&tables[t_roads]); - pgsql_pause_copy(&tables[t_line]); - pgsql_pause_copy(&tables[t_poly]); - pgsql_exec(tables[t_roads].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_roads].name, -osm_id ); - if ( expire_tiles_from_db(tables[t_line].sql_conn, -osm_id) != 0) - pgsql_exec(tables[t_line].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_line].name, -osm_id ); - if ( expire_tiles_from_db(tables[t_poly].sql_conn, -osm_id) != 0) - pgsql_exec(tables[t_poly].sql_conn, PGRES_COMMAND_OK, "DELETE FROM %s WHERE osm_id = %" PRIdOSMID, tables[t_poly].name, -osm_id ); - return 0; -} - -static int pgsql_delete_relation(osmid_t osm_id) -{ - if( !Options->slim ) - { - fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); - exit_nicely(); - } - pgsql_delete_relation_from_output(osm_id); - Options->mid->relations_delete(osm_id); - return 0; -} - -/* Modify is slightly trickier. The basic idea is we simply delete the - * object and create it with the new parameters. Then we need to mark the - * objects that depend on this one */ -static int pgsql_modify_node(osmid_t osm_id, double lat, double lon, struct keyval *tags) -{ - if( !Options->slim ) - { - fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); - exit_nicely(); - } - pgsql_delete_node(osm_id); - pgsql_add_node(osm_id, lat, lon, tags); - Options->mid->node_changed(osm_id); - return 0; -} - -static int pgsql_modify_way(osmid_t osm_id, osmid_t *nodes, int node_count, struct keyval *tags) -{ - if( !Options->slim ) - { - fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); - exit_nicely(); - } - pgsql_delete_way(osm_id); - pgsql_add_way(osm_id, nodes, node_count, tags); - Options->mid->way_changed(osm_id); - return 0; -} - -static int pgsql_modify_relation(osmid_t osm_id, struct member *members, int member_count, struct keyval *tags) -{ - if( !Options->slim ) - { - fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); - exit_nicely(); - } - pgsql_delete_relation(osm_id); - pgsql_add_relation(osm_id, members, member_count, tags); - Options->mid->relation_changed(osm_id); - return 0; -} - -struct output_t out_pgsql = { - .start = pgsql_out_start, - .connect = pgsql_out_connect, - .stop = pgsql_out_stop, - .cleanup = pgsql_out_cleanup, - .close = pgsql_out_close, - .node_add = pgsql_add_node, - .way_add = pgsql_add_way, - .relation_add = pgsql_add_relation, - - .node_modify = pgsql_modify_node, - .way_modify = pgsql_modify_way, - .relation_modify = pgsql_modify_relation, - - .node_delete = pgsql_delete_node, - .way_delete = pgsql_delete_way, - .relation_delete = pgsql_delete_relation -}; diff -Nru osm2pgsql-0.82.0/output-pgsql.cpp osm2pgsql-0.88.1/output-pgsql.cpp --- osm2pgsql-0.82.0/output-pgsql.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-pgsql.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,783 @@ +/* Implements the mid-layer processing for osm2pgsql + * using several PostgreSQL tables + * + * This layer stores data read in from the planet.osm file + * and is then read by the backend processing code to + * emit the final geometry-enabled output formats +*/ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_PTHREAD +#include +#endif + +#include "osmtypes.hpp" +#include "reprojection.hpp" +#include "output-pgsql.hpp" +#include "options.hpp" +#include "middle.hpp" +#include "pgsql.hpp" +#include "expire-tiles.hpp" +#include "wildcmp.hpp" +#include "node-ram-cache.hpp" +#include "taginfo_impl.hpp" +#include "tagtransform.hpp" +#include "util.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* make the diagnostic information work with older versions of + * boost - the function signature changed at version 1.54. + */ +#if BOOST_VERSION >= 105400 +#define BOOST_DIAGNOSTIC_INFO(e) boost::diagnostic_information((e), true) +#else +#define BOOST_DIAGNOSTIC_INFO(e) boost::diagnostic_information((e)) +#endif + +#define SRID (reproj->project_getprojinfo()->srs) + +/* FIXME: Shouldn't malloc this all to begin with but call realloc() + as required. The program will most likely segfault if it reads a + style file with more styles than this */ +#define MAX_STYLES 1000 + +#define NUM_TABLES (output_pgsql_t::t_MAX) + +/* example from: pg_dump -F p -t planet_osm gis +COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, building, bridge, layer, way) FROM stdin; +17959841 \N \N \N \N \N \N \N bus_stop \N \N \N \N \N \N -\N 0101000020E610000030CCA462B6C3D4BF92998C9B38E04940 +17401934 The Horn \N \N \N \N \N \N \N \N pub \N \N \N \N -\N 0101000020E6100000C12FC937140FD5BFB4D2F4FB0CE04940 +... + +mine - 01 01000000 48424298424242424242424256427364 +psql - 01 01000020 E6100000 30CCA462B6C3D4BF92998C9B38E04940 + 01 01000020 E6100000 48424298424242424242424256427364 +0x2000_0000 = hasSRID, following 4 bytes = srid, not supported by geos WKBWriter +Workaround - output SRID=4326; +*/ + +int output_pgsql_t::pgsql_out_node(osmid_t id, const taglist_t &tags, double node_lat, double node_lon) +{ + taglist_t outtags; + if (m_tagtransform->filter_node_tags(tags, *m_export_list.get(), outtags)) + return 1; + + expire->from_bbox(node_lon, node_lat, node_lon, node_lat); + m_tables[t_point]->write_node(id, outtags, node_lat, node_lon); + + return 0; +} + + +/* +COPY planet_osm (osm_id, name, place, landuse, leisure, "natural", man_made, waterway, highway, railway, amenity, tourism, learning, bu +ilding, bridge, layer, way) FROM stdin; +198497 Bedford Road \N \N \N \N \N \N residential \N \N \N \N \N \N \N 0102000020E610000004000000452BF702B342D5BF1C60E63BF8DF49406B9C4D470037D5BF5471E316F3DF4940DFA815A6EF35D5BF9AE95E27F5DF4940B41EB +E4C1421D5BF24D06053E7DF4940 +212696 Oswald Road \N \N \N \N \N \N minor \N \N \N \N \N \N \N 0102000020E610000004000000467D923B6C22D5BFA359D93EE4DF4940B3976DA7AD11D5BF84BBB376DBDF4940997FF44D9A06D5BF4223D8B8FEDF49404D158C4AEA04D +5BF5BB39597FCDF4940 +*/ +int output_pgsql_t::pgsql_out_way(osmid_t id, const taglist_t &tags, const nodelist_t &nodes, int exists) +{ + int polygon = 0, roads = 0; + double split_at; + + /* If the flag says this object may exist already, delete it first */ + if (exists) { + pgsql_delete_way_from_output(id); + // TODO: this now only has an effect when called from the iterate_ways + // call-back, so we need some alternative way to trigger this within + // osmdata_t. + const idlist_t rel_ids = m_mid->relations_using_way(id); + for (idlist_t::const_iterator itr = rel_ids.begin(); + itr != rel_ids.end(); ++itr) { + rels_pending_tracker->mark(*itr); + } + } + + taglist_t outtags; + if (m_tagtransform->filter_way_tags(tags, &polygon, &roads, *m_export_list.get(), + outtags)) + return 0; + /* Split long ways after around 1 degree or 100km */ + if (m_options.projection->get_proj_id() == PROJ_LATLONG) + split_at = 1; + else + split_at = 100 * 1000; + + tag *areatag = 0; + geometry_builder::maybe_wkts_t wkts = builder.get_wkt_split(nodes, polygon, split_at); + for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt) { + /* FIXME: there should be a better way to detect polygons */ + if (boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")) { + expire->from_nodes_poly(nodes, id); + if ((wkt->area > 0.0) && m_enable_way_area) { + char tmp[32]; + snprintf(tmp, sizeof(tmp), "%g", wkt->area); + if (!areatag) { + outtags.push_dedupe(tag("way_area", tmp)); + areatag = outtags.find("way_area"); + } else + areatag->value = tmp; + } + m_tables[t_poly]->write_wkt(id, outtags, wkt->geom.c_str()); + } else { + expire->from_nodes_line(nodes); + m_tables[t_line]->write_wkt(id, outtags, wkt->geom.c_str()); + if (roads) + m_tables[t_roads]->write_wkt(id, outtags, wkt->geom.c_str()); + } + } + + return 0; +} + +int output_pgsql_t::pgsql_out_relation(osmid_t id, const taglist_t &rel_tags, + const multinodelist_t &xnodes, const multitaglist_t & xtags, + const idlist_t &xid, const rolelist_t &xrole, + bool pending) +{ + if (xnodes.empty()) + return 0; + + int roads = 0; + int make_polygon = 0; + int make_boundary = 0; + double split_at; + + std::vector members_superseeded(xnodes.size(), 0); + taglist_t outtags; + + //if its a route relation make_boundary and make_polygon will be false otherwise one or the other will be true + if (m_tagtransform->filter_rel_member_tags(rel_tags, xtags, xrole, + &(members_superseeded[0]), &make_boundary, &make_polygon, &roads, + *m_export_list.get(), outtags)) { + return 0; + } + + /* Split long linear ways after around 1 degree or 100km (polygons not effected) */ + if (m_options.projection->get_proj_id() == PROJ_LATLONG) + split_at = 1; + else + split_at = 100 * 1000; + + //this will either make lines or polygons (unless the lines arent a ring or are less than 3 pts) depending on the tag transform above + //TODO: pick one or the other based on which we expect to care about + geometry_builder::maybe_wkts_t wkts = builder.build_both(xnodes, make_polygon, m_options.enable_multi, split_at, id); + + if (!wkts->size()) { + return 0; + } + + tag *areatag = 0; + for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt) + { + expire->from_wkt(wkt->geom.c_str(), -id); + /* FIXME: there should be a better way to detect polygons */ + if (boost::starts_with(wkt->geom, "POLYGON") || boost::starts_with(wkt->geom, "MULTIPOLYGON")) { + if ((wkt->area > 0.0) && m_enable_way_area) { + char tmp[32]; + snprintf(tmp, sizeof(tmp), "%g", wkt->area); + if (!areatag) { + outtags.push_dedupe(tag("way_area", tmp)); + areatag = outtags.find("way_area"); + } else + areatag->value = tmp; + } + m_tables[t_poly]->write_wkt(-id, outtags, wkt->geom.c_str()); + } else { + m_tables[t_line]->write_wkt(-id, outtags, wkt->geom.c_str()); + if (roads) + m_tables[t_roads]->write_wkt(-id, outtags, wkt->geom.c_str()); + } + } + + /* Tagtransform will have marked those member ways of the relation that + * have fully been dealt with as part of the multi-polygon entry. + * Set them in the database as done and delete their entry to not + * have duplicates */ + //dont do this when working with pending relations as its not needed + if (make_polygon) { + for (size_t i=0; i < xid.size(); i++) { + if (members_superseeded[i]) { + pgsql_delete_way_from_output(xid[i]); + if(!pending) + ways_done_tracker->mark(xid[i]); + } + } + } + + // If the tag transform said the polygon looked like a boundary we want to make that as well + // If we are making a boundary then also try adding any relations which form complete rings + // The linear variants will have already been processed above + if (make_boundary) { + tag *areatag = 0; + wkts = builder.build_polygons(xnodes, m_options.enable_multi, id); + for(geometry_builder::wkt_itr wkt = wkts->begin(); wkt != wkts->end(); ++wkt) + { + expire->from_wkt(wkt->geom.c_str(), -id); + if ((wkt->area > 0.0) && m_enable_way_area) { + char tmp[32]; + snprintf(tmp, sizeof(tmp), "%g", wkt->area); + if (!areatag) { + outtags.push_dedupe(tag("way_area", tmp)); + areatag = outtags.find("way_area"); + } else + areatag->value = tmp; + } + m_tables[t_poly]->write_wkt(-id, outtags, wkt->geom.c_str()); + } + } + + return 0; +} + + + +namespace { +/* Using pthreads requires us to shoe-horn everything into various void* + * pointers. Improvement for the future: just use boost::thread. */ +struct pthread_thunk { + table_t *ptr; + boost::exception_ptr error; +}; + +extern "C" void *pthread_output_pgsql_stop_one(void *arg) { + pthread_thunk *thunk = static_cast(arg); + + try { + thunk->ptr->stop(); + + } catch (...) { + thunk->error = boost::current_exception(); + } + + return NULL; +} +} // anonymous namespace + +void output_pgsql_t::enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { + osmid_t const prev = ways_pending_tracker->last_returned(); + if (id_tracker::is_valid(prev) && prev >= id) { + if (prev > id) { + job_queue.push(pending_job_t(id, output_id)); + } + // already done the job + return; + } + + //make sure we get the one passed in + if(!ways_done_tracker->is_marked(id) && id_tracker::is_valid(id)) { + job_queue.push(pending_job_t(id, output_id)); + added++; + } + + //grab the first one or bail if its not valid + osmid_t popped = ways_pending_tracker->pop_mark(); + if(!id_tracker::is_valid(popped)) + return; + + //get all the ones up to the id that was passed in + while (popped < id) { + if (!ways_done_tracker->is_marked(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + popped = ways_pending_tracker->pop_mark(); + } + + //make sure to get this one as well and move to the next + if(popped > id) { + if (!ways_done_tracker->is_marked(popped) && id_tracker::is_valid(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + } +} + +int output_pgsql_t::pending_way(osmid_t id, int exists) { + taglist_t tags_int; + nodelist_t nodes_int; + int ret = 0; + + // Try to fetch the way from the DB + if (!m_mid->ways_get(id, tags_int, nodes_int)) { + // Output the way + //ret = reprocess_way(id, nodes_int, count_int, &tags_int, exists); + ret = pgsql_out_way(id, tags_int, nodes_int, exists); + } + + return ret; +} + +void output_pgsql_t::enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { + osmid_t const prev = rels_pending_tracker->last_returned(); + if (id_tracker::is_valid(prev) && prev >= id) { + if (prev > id) { + job_queue.push(pending_job_t(id, output_id)); + } + // already done the job + return; + } + + //make sure we get the one passed in + if(id_tracker::is_valid(id)) { + job_queue.push(pending_job_t(id, output_id)); + added++; + } + + //grab the first one or bail if its not valid + osmid_t popped = rels_pending_tracker->pop_mark(); + if(!id_tracker::is_valid(popped)) + return; + + //get all the ones up to the id that was passed in + while (popped < id) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + popped = rels_pending_tracker->pop_mark(); + } + + //make sure to get this one as well and move to the next + if(popped > id) { + if(id_tracker::is_valid(popped)) { + job_queue.push(pending_job_t(popped, output_id)); + added++; + } + } +} + +int output_pgsql_t::pending_relation(osmid_t id, int exists) { + taglist_t tags_int; + memberlist_t members_int; + int ret = 0; + + // Try to fetch the relation from the DB + if (!m_mid->relations_get(id, members_int, tags_int)) { + ret = pgsql_process_relation(id, members_int, tags_int, exists, true); + } + + return ret; +} + +void output_pgsql_t::commit() +{ + for (int i=0; icommit(); + } +} + +void output_pgsql_t::stop() +{ + int i; +#ifdef HAVE_PTHREAD + pthread_t threads[NUM_TABLES]; +#endif + +#ifdef HAVE_PTHREAD + if (m_options.parallel_indexing) { + pthread_thunk thunks[NUM_TABLES]; + for (i=0; istop(); + +#ifdef HAVE_PTHREAD + } +#endif + + expire->output_and_destroy(); + expire.reset(); +} + +int output_pgsql_t::node_add(osmid_t id, double lat, double lon, const taglist_t &tags) +{ + pgsql_out_node(id, tags, lat, lon); + + return 0; +} + +int output_pgsql_t::way_add(osmid_t id, const idlist_t &nds, const taglist_t &tags) +{ + int polygon = 0; + int roads = 0; + taglist_t outtags; + + /* Check whether the way is: (1) Exportable, (2) Maybe a polygon */ + int filter = m_tagtransform->filter_way_tags(tags, &polygon, &roads, *m_export_list.get(), outtags); + + /* If this isn't a polygon then it can not be part of a multipolygon + Hence only polygons are "pending" */ + if (!filter && polygon) { ways_pending_tracker->mark(id); } + + if( !polygon && !filter ) + { + /* Get actual node data and generate output */ + nodelist_t nodes; + m_mid->nodes_get_list(nodes, nds); + pgsql_out_way(id, outtags, nodes, 0); + } + return 0; +} + + +/* This is the workhorse of pgsql_add_relation, split out because it is used as the callback for iterate relations */ +int output_pgsql_t::pgsql_process_relation(osmid_t id, const memberlist_t &members, + const taglist_t &tags, int exists, bool pending) +{ + /* If the flag says this object may exist already, delete it first */ + if(exists) + pgsql_delete_relation_from_output(id); + + taglist_t outtags; + + if (m_tagtransform->filter_rel_tags(tags, *m_export_list.get(), outtags)) + return 1; + + idlist_t xid2; + multitaglist_t xtags2; + multinodelist_t xnodes; + + for (memberlist_t::const_iterator it = members.begin(); it != members.end(); ++it) + { + /* Need to handle more than just ways... */ + if (it->type == OSMTYPE_WAY) + xid2.push_back(it->id); + } + + idlist_t xid; + m_mid->ways_get_list(xid2, xid, xtags2, xnodes); + int polygon = 0, roads = 0; + multitaglist_t xtags(xid.size(), taglist_t()); + rolelist_t xrole(xid.size(), 0); + + for (size_t i = 0; i < xid.size(); i++) { + for (size_t j = i; j < members.size(); j++) { + if (members[j].id == xid[i]) { + //filter the tags on this member because we got it from the middle + //and since the middle is no longer tied to the output it no longer + //shares any kind of tag transform and therefore all original tags + //will come back and need to be filtered by individual outputs before + //using these ways + m_tagtransform->filter_way_tags(xtags2[i], &polygon, &roads, + *m_export_list.get(), xtags[i]); + //TODO: if the filter says that this member is now not interesting we + //should decrement the count and remove his nodes and tags etc. for + //now we'll just keep him with no tags so he will get filtered later + xrole[i] = &members[j].role; + break; + } + } + } + + /* At some point we might want to consider storing the retrieved data in the members, rather than as separate arrays */ + pgsql_out_relation(id, outtags, xnodes, xtags, xid, xrole, pending); + + return 0; +} + +int output_pgsql_t::relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) +{ + const std::string *type = tags.get("type"); + + /* Must have a type field or we ignore it */ + if (!type) + return 0; + + /* Only a limited subset of type= is supported, ignore other */ + if ( (*type != "route") && (*type != "multipolygon") && (*type != "boundary")) + return 0; + + + return pgsql_process_relation(id, members, tags, 0); +} + +/* Delete is easy, just remove all traces of this object. We don't need to + * worry about finding objects that depend on it, since the same diff must + * contain the change for that also. */ +int output_pgsql_t::node_delete(osmid_t osm_id) +{ + if( !m_options.slim ) + { + fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); + util::exit_nicely(); + } + + if ( expire->from_db(m_tables[t_point].get(), osm_id) != 0) + m_tables[t_point]->delete_row(osm_id); + + return 0; +} + +/* Seperated out because we use it elsewhere */ +int output_pgsql_t::pgsql_delete_way_from_output(osmid_t osm_id) +{ + /* Optimisation: we only need this is slim mode */ + if( !m_options.slim ) + return 0; + /* in droptemp mode we don't have indices and this takes ages. */ + if (m_options.droptemp) + return 0; + + m_tables[t_roads]->delete_row(osm_id); + if ( expire->from_db(m_tables[t_line].get(), osm_id) != 0) + m_tables[t_line]->delete_row(osm_id); + if ( expire->from_db(m_tables[t_poly].get(), osm_id) != 0) + m_tables[t_poly]->delete_row(osm_id); + return 0; +} + +int output_pgsql_t::way_delete(osmid_t osm_id) +{ + if( !m_options.slim ) + { + fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); + util::exit_nicely(); + } + pgsql_delete_way_from_output(osm_id); + return 0; +} + +/* Relations are identified by using negative IDs */ +int output_pgsql_t::pgsql_delete_relation_from_output(osmid_t osm_id) +{ + m_tables[t_roads]->delete_row(-osm_id); + if ( expire->from_db(m_tables[t_line].get(), -osm_id) != 0) + m_tables[t_line]->delete_row(-osm_id); + if ( expire->from_db(m_tables[t_poly].get(), -osm_id) != 0) + m_tables[t_poly]->delete_row(-osm_id); + return 0; +} + +int output_pgsql_t::relation_delete(osmid_t osm_id) +{ + if( !m_options.slim ) + { + fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); + util::exit_nicely(); + } + pgsql_delete_relation_from_output(osm_id); + return 0; +} + +/* Modify is slightly trickier. The basic idea is we simply delete the + * object and create it with the new parameters. Then we need to mark the + * objects that depend on this one */ +int output_pgsql_t::node_modify(osmid_t osm_id, double lat, double lon, const taglist_t &tags) +{ + if( !m_options.slim ) + { + fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); + util::exit_nicely(); + } + node_delete(osm_id); + node_add(osm_id, lat, lon, tags); + return 0; +} + +int output_pgsql_t::way_modify(osmid_t osm_id, const idlist_t &nodes, const taglist_t &tags) +{ + if( !m_options.slim ) + { + fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); + util::exit_nicely(); + } + way_delete(osm_id); + way_add(osm_id, nodes, tags); + + return 0; +} + +int output_pgsql_t::relation_modify(osmid_t osm_id, const memberlist_t &members, const taglist_t &tags) +{ + if( !m_options.slim ) + { + fprintf( stderr, "Cannot apply diffs unless in slim mode\n" ); + util::exit_nicely(); + } + relation_delete(osm_id); + relation_add(osm_id, members, tags); + return 0; +} + +int output_pgsql_t::start() +{ + for(std::vector >::iterator table = m_tables.begin(); table != m_tables.end(); ++table) + { + //setup the table in postgres + table->get()->start(); + } + + return 0; +} + +boost::shared_ptr output_pgsql_t::clone(const middle_query_t* cloned_middle) const { + output_pgsql_t *clone = new output_pgsql_t(*this); + clone->m_mid = cloned_middle; + //NOTE: we need to know which ways were used by relations so each thread + //must have a copy of the original marked done ways, its read only so its ok + clone->ways_done_tracker = ways_done_tracker; + return boost::shared_ptr(clone); +} + +output_pgsql_t::output_pgsql_t(const middle_query_t* mid_, const options_t &options_) + : output_t(mid_, options_), + ways_pending_tracker(new id_tracker()), + ways_done_tracker(new id_tracker()), + rels_pending_tracker(new id_tracker()) { + + reproj = m_options.projection; + builder.set_exclude_broken_polygon(m_options.excludepoly); + + m_export_list.reset(new export_list()); + + m_enable_way_area = read_style_file( m_options.style, m_export_list.get() ); + + try { + m_tagtransform.reset(new tagtransform(&m_options)); + } + catch(const std::runtime_error& e) { + fprintf(stderr, "%s\n", e.what()); + fprintf(stderr, "Error: Failed to initialise tag processing.\n"); + util::exit_nicely(); + } + + expire.reset(new expire_tiles(&m_options)); + + //for each table + m_tables.reserve(NUM_TABLES); + for (int i=0; inormal_columns((i == t_point)?OSMTYPE_NODE:OSMTYPE_WAY); + + //figure out what name we are using for this and what type + std::string name = m_options.prefix; + std::string type; + switch(i) + { + case t_point: + name += "_point"; + type = "POINT"; + break; + case t_line: + name += "_line"; + type = "LINESTRING"; + break; + case t_poly: + name += "_polygon"; + type = "GEOMETRY"; // Actually POLGYON & MULTIPOLYGON but no way to limit to just these two + break; + case t_roads: + name += "_roads"; + type = "LINESTRING"; + break; + default: + //TODO: error message about coding error + util::exit_nicely(); + } + + //tremble in awe of this massive constructor! seriously we are trying to avoid passing an + //options object because we want to make use of the table_t in output_mutli_t which could + //have a different tablespace/hstores/etc per table + m_tables.push_back(boost::shared_ptr( + new table_t( + m_options.conninfo, name, type, columns, m_options.hstore_columns, SRID, + m_options.append, m_options.slim, m_options.droptemp, m_options.hstore_mode, + m_options.enable_hstore_index, m_options.tblsmain_data, m_options.tblsmain_index + ) + )); + } +} + +output_pgsql_t::output_pgsql_t(const output_pgsql_t& other): + output_t(other.m_mid, other.m_options), m_tagtransform(new tagtransform(&m_options)), m_enable_way_area(other.m_enable_way_area), + m_export_list(new export_list(*other.m_export_list)), reproj(other.reproj), + expire(new expire_tiles(&m_options)), + ways_pending_tracker(new id_tracker()), ways_done_tracker(new id_tracker()), rels_pending_tracker(new id_tracker()) +{ + builder.set_exclude_broken_polygon(m_options.excludepoly); + for(std::vector >::const_iterator t = other.m_tables.begin(); t != other.m_tables.end(); ++t) { + //copy constructor will just connect to the already there table + m_tables.push_back(boost::shared_ptr(new table_t(**t))); + } +} + +output_pgsql_t::~output_pgsql_t() { +} + +size_t output_pgsql_t::pending_count() const { + return ways_pending_tracker->size() + rels_pending_tracker->size(); +} + +void output_pgsql_t::merge_pending_relations(boost::shared_ptr other) { + boost::shared_ptr tracker = other->get_pending_relations(); + osmid_t id; + while(tracker.get() && id_tracker::is_valid((id = tracker->pop_mark()))){ + rels_pending_tracker->mark(id); + } +} +void output_pgsql_t::merge_expire_trees(boost::shared_ptr other) { + if(other->get_expire_tree().get()) + expire->merge_and_destroy(*other->get_expire_tree()); +} + +boost::shared_ptr output_pgsql_t::get_pending_relations() { + return rels_pending_tracker; +} +boost::shared_ptr output_pgsql_t::get_expire_tree() { + return expire; +} diff -Nru osm2pgsql-0.82.0/output-pgsql.h osm2pgsql-0.88.1/output-pgsql.h --- osm2pgsql-0.82.0/output-pgsql.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/output-pgsql.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,13 +0,0 @@ -/* Implements the output-layer processing for osm2pgsql - * storing the data in several PostgreSQL tables - * with the final PostGIS geometries for each entity -*/ - -#ifndef OUTPUT_PGSQL_H -#define OUTPUT_PGSQL_H - -#include "output.h" - -extern struct output_t out_pgsql; - -#endif diff -Nru osm2pgsql-0.82.0/output-pgsql.hpp osm2pgsql-0.88.1/output-pgsql.hpp --- osm2pgsql-0.82.0/output-pgsql.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/output-pgsql.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,92 @@ +/* Implements the output-layer processing for osm2pgsql + * storing the data in several PostgreSQL tables + * with the final PostGIS geometries for each entity +*/ + +#ifndef OUTPUT_PGSQL_H +#define OUTPUT_PGSQL_H + +#include "output.hpp" +#include "tagtransform.hpp" +#include "geometry-builder.hpp" +#include "reprojection.hpp" +#include "expire-tiles.hpp" +#include "id-tracker.hpp" +#include "table.hpp" + +#include +#include + +class output_pgsql_t : public output_t { +public: + enum table_id { + t_point = 0, t_line, t_poly, t_roads, t_MAX + }; + + output_pgsql_t(const middle_query_t* mid_, const options_t &options_); + virtual ~output_pgsql_t(); + output_pgsql_t(const output_pgsql_t& other); + + virtual boost::shared_ptr clone(const middle_query_t* cloned_middle) const; + + int start(); + void stop(); + void commit(); + + void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added); + int pending_way(osmid_t id, int exists); + + void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added); + int pending_relation(osmid_t id, int exists); + + int node_add(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_add(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags); + int way_modify(osmid_t id, const idlist_t &nodes, const taglist_t &tags); + int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags); + + int node_delete(osmid_t id); + int way_delete(osmid_t id); + int relation_delete(osmid_t id); + + size_t pending_count() const; + + void merge_pending_relations(boost::shared_ptr other); + void merge_expire_trees(boost::shared_ptr other); + virtual boost::shared_ptr get_pending_relations(); + virtual boost::shared_ptr get_expire_tree(); + +protected: + + int pgsql_out_node(osmid_t id, const taglist_t &tags, double node_lat, double node_lon); + int pgsql_out_way(osmid_t id, const taglist_t &tags, const nodelist_t &nodes, int exists); + int pgsql_out_relation(osmid_t id, const taglist_t &rel_tags, + const multinodelist_t &xnodes, const multitaglist_t & xtags, + const idlist_t &xid, const rolelist_t &xrole, + bool pending); + int pgsql_process_relation(osmid_t id, const memberlist_t &members, const taglist_t &tags, int exists, bool pending=false); + int pgsql_delete_way_from_output(osmid_t osm_id); + int pgsql_delete_relation_from_output(osmid_t osm_id); + + boost::scoped_ptr m_tagtransform; + + //enable output of a generated way_area tag to either hstore or its own column + int m_enable_way_area; + + std::vector > m_tables; + + boost::scoped_ptr m_export_list; + + geometry_builder builder; + + boost::shared_ptr reproj; + boost::shared_ptr expire; + + boost::shared_ptr ways_pending_tracker, ways_done_tracker, rels_pending_tracker; + + const static std::string NAME; +}; + +#endif diff -Nru osm2pgsql-0.82.0/parse.cpp osm2pgsql-0.88.1/parse.cpp --- osm2pgsql-0.82.0/parse.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,178 @@ +#include "config.h" +#include "parse.hpp" +#include "parse-o5m.hpp" +#ifdef BUILD_READER_PBF +# include "parse-pbf.hpp" +#endif +#include "parse-xml2.hpp" + +#include +#include +#include +#include +#include + + +#define INIT_MAX_MEMBERS 64 +#define INIT_MAX_NODES 4096 + +#ifdef _MSC_VER +#define strcasecmp _stricmp +#endif + + +parse_delegate_t::parse_delegate_t(const int extra_attributes, + const boost::optional &bbox, + boost::shared_ptr projection): +m_extra_attributes(extra_attributes), m_proj(projection), m_count_node(0), m_max_node(0), +m_count_way(0), m_max_way(0), m_count_rel(0), m_max_rel(0), m_start_node(0), m_start_way(0), m_start_rel(0) +{ + m_bbox = bool(bbox); + if (m_bbox) + parse_bbox(*bbox); +} + +parse_delegate_t::~parse_delegate_t() +{ +} + +int parse_delegate_t::streamFile(const char* input_reader, const char* filename,const int sanitize, osmdata_t *osmdata) +{ + //process the input file with the right parser + parse_t* parser = get_input_reader(input_reader, filename); + int ret = parser->streamFile(filename, sanitize, osmdata); + + //update statisics + m_count_node += parser->count_node; + m_count_way += parser->count_way; + m_count_rel += parser->count_rel; + m_max_node = std::max(parser->max_node, m_max_node); + m_max_way = std::max(parser->max_way, m_max_way); + m_max_rel = std::max(parser->max_rel, m_max_rel); + m_start_node = m_start_node == 0 ? parser->start_node : m_start_node; + m_start_way = m_start_way == 0 ? parser->start_way : m_start_way; + m_start_rel = m_start_rel == 0 ? parser->start_rel : m_start_rel; + + //done + delete parser; + + return ret; +} +void parse_delegate_t::printSummary() const +{ + time_t now = time(NULL); + time_t end_nodes = m_start_way > 0 ? m_start_way : now; + time_t end_way = m_start_rel > 0 ? m_start_rel : now; + time_t end_rel = now; + + fprintf(stderr, + "Node stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", + m_count_node, m_max_node, + m_count_node > 0 ? (int) (end_nodes - m_start_node) : 0); + fprintf(stderr, + "Way stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", + m_count_way, m_max_way, + m_count_way > 0 ? (int) (end_way - m_start_way) : 0); + fprintf(stderr, + "Relation stats: total(%" PRIdOSMID "), max(%" PRIdOSMID ") in %is\n", + m_count_rel, m_max_rel, + m_count_rel > 0 ? (int) (end_rel - m_start_rel) : 0); +} + +boost::shared_ptr parse_delegate_t::getProjection() const +{ + return m_proj; +} + +void parse_delegate_t::parse_bbox(const std::string &bbox_) +{ + int n = sscanf(bbox_.c_str(), "%lf,%lf,%lf,%lf", &(m_minlon), &(m_minlat), &(m_maxlon), &(m_maxlat)); + if (n != 4) + throw std::runtime_error("Bounding box must be specified like: minlon,minlat,maxlon,maxlat\n"); + + if (m_maxlon <= m_minlon) + throw std::runtime_error("Bounding box failed due to maxlon <= minlon\n"); + + if (m_maxlat <= m_minlat) + throw std::runtime_error("Bounding box failed due to maxlat <= minlat\n"); + + fprintf(stderr, "Applying Bounding box: %f,%f to %f,%f\n", m_minlon, m_minlat, m_maxlon, m_maxlat); +} + +parse_t* parse_delegate_t::get_input_reader(const char* input_reader, const char* filename) +{ + // if input_reader is forced to a specific iput format + if (strcmp("auto", input_reader) != 0) { + if (strcmp("libxml2", input_reader) == 0) { + return new parse_xml2_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat); + } else if (strcmp("primitive", input_reader) == 0) { + // The more robust libxml2 parser can be used instead of primitive + return new parse_xml2_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat); +#ifdef BUILD_READER_PBF + } else if (strcmp("pbf", input_reader) == 0) { + return new parse_pbf_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat); +#endif + } else if (strcmp("o5m", input_reader) == 0) { + return new parse_o5m_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat); + } else { + fprintf(stderr, "Input parser `%s' not recognised. Should be one of [libxml2, o5m" +#ifdef BUILD_READER_PBF + ", pbf" +#endif + "].\n", input_reader); + exit(EXIT_FAILURE); + } + } // if input_reader is not forced by -r switch try to auto-detect it by file extension + else { + if (strcasecmp(".pbf", filename + strlen(filename) - 4) == 0) { +#ifdef BUILD_READER_PBF + return new parse_pbf_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat); +#else + fprintf(stderr, "ERROR: PBF support has not been compiled into this version of osm2pgsql, please either compile it with pbf support or use one of the other input formats\n"); + exit(EXIT_FAILURE); +#endif + } else if (strcasecmp(".o5m", filename + strlen(filename) - 4) == 0 + || strcasecmp(".o5c", filename + strlen(filename) - 4) == 0) { + return new parse_o5m_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat); + } else { + return new parse_xml2_t(m_extra_attributes, m_bbox, m_proj, m_minlon, m_minlat, m_maxlon, m_maxlat); + } + } +} + + + +parse_t::parse_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon_, const double minlat_, const double maxlon_, const double maxlat_): + extra_attributes(extra_attributes_), bbox(bbox_), minlon(minlon_), minlat(minlat_), + maxlon(maxlon_), maxlat(maxlat_), proj(projection_) +{ + osm_id = count_node = max_node = count_way = max_way = count_rel = 0; + max_rel = parallel_indexing = start_node = start_way = start_rel = 0; + node_lon = node_lat = 0; + + filetype = FILETYPE_NONE; + action = ACTION_NONE; +} + +parse_t::~parse_t() +{ +} + + +void parse_t::print_status() +{ + time_t now = time(NULL); + time_t end_nodes = start_way > 0 ? start_way : now; + time_t end_way = start_rel > 0 ? start_rel : now; + time_t end_rel = now; + fprintf(stderr, + "\rProcessing: Node(%" PRIdOSMID "k %.1fk/s) Way(%" PRIdOSMID "k %.2fk/s) Relation(%" PRIdOSMID " %.2f/s)", + count_node / 1000, + (double) count_node / 1000.0 / ((int) (end_nodes - start_node) > 0 ? (double) (end_nodes - start_node) : 1.0), + count_way / 1000, + count_way > 0 ? (double) count_way / 1000.0 / ((double) (end_way - start_way) > 0.0 ? (double) (end_way - start_way) : 1.0) : 0.0, count_rel, + count_rel > 0 ? (double) count_rel / ((double) (end_rel - start_rel) > 0.0 ? (double) (end_rel - start_rel) : 1.0) : 0.0); +} + + diff -Nru osm2pgsql-0.82.0/parse.hpp osm2pgsql-0.88.1/parse.hpp --- osm2pgsql-0.82.0/parse.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,96 @@ +#ifndef PARSE_H +#define PARSE_H + +#include "osmtypes.hpp" + +#include +#include +#include +#include + +typedef enum { FILETYPE_NONE, FILETYPE_OSM, FILETYPE_OSMCHANGE, FILETYPE_PLANETDIFF } filetypes_t; +typedef enum { ACTION_NONE, ACTION_CREATE, ACTION_MODIFY, ACTION_DELETE } actions_t; + +class parse_t; +class osmdata_t; +struct reprojection; + +class parse_delegate_t +{ +public: + parse_delegate_t(const int extra_attributes, const boost::optional &bbox, boost::shared_ptr projection); + ~parse_delegate_t(); + + int streamFile(const char* input_reader, const char* filename, const int sanitize, osmdata_t *osmdata); + void printSummary() const; + boost::shared_ptr getProjection() const; + +private: + parse_delegate_t(); + void parse_bbox(const std::string &bbox); + parse_t* get_input_reader(const char* input_reader, const char* filename); + + const int m_extra_attributes; + boost::shared_ptr m_proj; + osmid_t m_count_node, m_max_node; + osmid_t m_count_way, m_max_way; + osmid_t m_count_rel, m_max_rel; + time_t m_start_node, m_start_way, m_start_rel; + + bool m_bbox; + double m_minlon, m_minlat, m_maxlon, m_maxlat; +}; + +class parse_t +{ + friend class parse_delegate_t; + +public: + parse_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon, const double minlat, const double maxlon, const double maxlat); + virtual ~parse_t(); + + virtual int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata) = 0; + +protected: + parse_t(); + + void print_status(); + + int node_wanted(double lat, double lon) + { + if (!bbox) + return 1; + + if (lat < minlat || lat > maxlat) + return 0; + if (lon < minlon || lon > maxlon) + return 0; + return 1; + } + + osmid_t count_node, max_node; + osmid_t count_way, max_way; + osmid_t count_rel, max_rel; + time_t start_node, start_way, start_rel; + + /* Since {node,way} elements are not nested we can guarantee the + values in an end tag must match those of the corresponding + start tag and can therefore be cached. + */ + double node_lon, node_lat; + taglist_t tags; + idlist_t nds; + memberlist_t members; + osmid_t osm_id; + filetypes_t filetype; + actions_t action; + int parallel_indexing; + + const int extra_attributes; + mutable bool bbox; + mutable double minlon, minlat, maxlon, maxlat; + const boost::shared_ptr proj; +}; + +#endif diff -Nru osm2pgsql-0.82.0/parse-o5m.c osm2pgsql-0.88.1/parse-o5m.c --- osm2pgsql-0.82.0/parse-o5m.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-o5m.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,954 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ -#define _GNU_SOURCE - -/* 2011-07-03 02:30 - Markus Weber */ - -#include -#include -#include -#include -#include -#include -#include - -#include "osmtypes.h" -#include "reprojection.h" -#include "output.h" - -#define inline - -typedef enum {false= 0,true= 1} bool; -typedef uint8_t byte; -typedef unsigned int uint; -#define isdig(x) isdigit((unsigned char)(x)) -static int loglevel= 0; /* logging to stderr; */ -/* 0: no logging; 1: small logging; 2: normal logging; - 3: extended logging; */ -#define DP(f) fprintf(stderr,"- Debug: " #f "\n"); -#define DPv(f,...) fprintf(stderr,"- Debug: " #f "\n",__VA_ARGS__); -#if __WIN32__ -#define NL "\r\n" /* use CR/LF as new-line sequence */ - #define off_t off64_t - #define lseek lseek64 -#else -#define NL "\n" /* use LF as new-line sequence */ - #define O_BINARY 0 -#endif - -#define PERR(f) \ - fprintf(stderr,"osm2pgsql Error: " f "\n"); -/* print error message */ -#define PERRv(f,...) \ - fprintf(stderr,"osm2pgsql Error: " f "\n",__VA_ARGS__); -/* print error message with value(s) */ -#define WARN(f) { static int msgn= 3; if(--msgn>=0) \ - fprintf(stderr,"osm2pgsql Warning: " f "\n"); } -/* print a warning message, do it maximal 3 times */ -#define WARNv(f,...) { static int msgn= 3; if(--msgn>=0) \ - fprintf(stderr,"osm2pgsql Warning: " f "\n",__VA_ARGS__); } -/* print a warning message with value(s), do it maximal 3 times */ -#define PINFO(f) \ - fprintf(stderr,"osm2pgsql: " f "\n"); -/* print info message */ -#define ONAME(i) \ - (i==0? "node": i==1? "way": i==2? "relation": "unknown object") - -static inline char *stpcpy0(char *dest, const char *src) { - /* redefinition of C99's stpcpy() because it's missing in MinGW, - and declaration in Linux seems to be wrong; */ - while(*src!=0) - *dest++= *src++; - *dest= 0; - return dest; -} /* end stpcpy0() */ - -static inline char* uint32toa(uint32_t v,char* s) { - /* convert uint32_t integer into string; - v: long integer value to convert; - return: s; - s[]: digit string; */ - char* s1,*s2; - char c; - - s1= s; - if(v==0) - *s1++= '0'; - s2= s1; - while(v>0) - { *s2++= "0123456789"[v%10]; v/= 10; } - *s2--= 0; - while(s2>s1) - { c= *s1; *s1= *s2; *s2= c; s1++; s2--; } - return s; -} /* end uint32toa() */ - -static inline void createtimestamp(uint64_t v,char* sp) { - /* write a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z", - into a string; - v: value of the timestamp; - sp[21]: destination string; */ - time_t vtime; - struct tm tm; - int i; - - vtime= v; - #if __WIN32__ - memcpy(&tm,gmtime(&vtime),sizeof(tm)); - #else - gmtime_r(&vtime,&tm); - #endif - i= tm.tm_year+1900; - sp+= 3; *sp--= i%10+'0'; - i/=10; *sp--= i%10+'0'; - i/=10; *sp--= i%10+'0'; - i/=10; *sp= i%10+'0'; - sp+= 4; *sp++= '-'; - i= tm.tm_mon+1; - *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= '-'; - i= tm.tm_mday; - *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'T'; - i= tm.tm_hour; - *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; - i= tm.tm_min; - *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; - i= tm.tm_sec%60; - *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'Z'; *sp= 0; -} /* end createtimestamp() */ - - - -/*------------------------------------------------------------ - Module pbf_ protobuf conversions module - ------------------------------------------------------------ - - this module provides procedures for conversions from - protobuf formats to regular numbers; - as usual, all identifiers of a module have the same prefix, - in this case 'pbf'; one underline will follow in case of a - global accessible object, two underlines in case of objects - which are not meant to be accessed from outside this module; - the sections of private and public definitions are separated - by a horizontal line: ---- - many procedures have a parameter 'pp'; here, the address of - a buffer pointer is expected; this pointer will be incremented - by the number of bytes the converted protobuf element consumes; - - ------------------------------------------------------------ */ - -static inline uint32_t pbf_uint32(byte** pp) { - /* get the value of an unsigned integer; - pp: see module header; */ - byte* p; - uint32_t i; - uint32_t fac; - - p= *pp; - i= *p; - if((*p & 0x80)==0) { /* just one byte */ - (*pp)++; -return i; - } - i&= 0x7f; - fac= 0x80; - while(*++p & 0x80) { /* more byte(s) will follow */ - i+= (*p & 0x7f)*fac; - fac<<= 7; - } - i+= *p++ *fac; - *pp= p; - return i; -} /* end pbf_uint32() */ - -static inline int32_t pbf_sint32(byte** pp) { - /* get the value of an unsigned integer; - pp: see module header; */ - byte* p; - int32_t i; - int32_t fac; - int sig; - - p= *pp; - i= *p; - if((*p & 0x80)==0) { /* just one byte */ - (*pp)++; - if(i & 1) /* negative */ -return -1-(i>>1); - else -return i>>1; - } - sig= i & 1; - i= (i & 0x7e)>>1; - fac= 0x40; - while(*++p & 0x80) { /* more byte(s) will follow */ - i+= (*p & 0x7f)*fac; - fac<<= 7; - } - i+= *p++ *fac; - *pp= p; - if(sig) /* negative */ -return -1-i; - else -return i; -} /* end pbf_sint32() */ - -static inline uint64_t pbf_uint64(byte** pp) { - /* get the value of an unsigned integer; - pp: see module header; */ - byte* p; - uint64_t i; - uint64_t fac; - - p= *pp; - i= *p; - if((*p & 0x80)==0) { /* just one byte */ - (*pp)++; -return i; - } - i&= 0x7f; - fac= 0x80; - while(*++p & 0x80) { /* more byte(s) will follow */ - i+= (*p & 0x7f)*fac; - fac<<= 7; - } - i+= *p++ *fac; - *pp= p; - return i; -} /* end pbf_uint64() */ - -static inline int64_t pbf_sint64(byte** pp) { - /* get the value of a signed integer; - pp: see module header; */ - byte* p; - int64_t i; - int64_t fac; - int sig; - - p= *pp; - i= *p; - if((*p & 0x80)==0) { /* just one byte */ - (*pp)++; - if(i & 1) /* negative */ -return -1-(i>>1); - else -return i>>1; - } - sig= i & 1; - i= (i & 0x7e)>>1; - fac= 0x40; - while(*++p & 0x80) { /* more byte(s) will follow */ - i+= (*p & 0x7f)*fac; - fac<<= 7; - } - i+= *p++ *fac; - *pp= p; - if(sig) /* negative */ -return -1-i; - else -return i; -} /* end pbf_sint64() */ - -#if 0 /* not used at present */ -static inline void pbf_intjump(byte** pp) { - /* jump over a protobuf formatted integer; - pp: see module header; - we do not care about a possibly existing identifier, - therefore as the start address *pp the address of the - integer value is expected; */ - byte* p; - - p= *pp; - while(*p & 0x80) p++; p++; - *pp= p; -} /* end pbf_intjump() */ -#endif - -/*------------------------------------------------------------ - end Module pbf_ protobuf conversions module - ------------------------------------------------------------ */ - - - -/*------------------------------------------------------------ - Module read_ OSM file read module - ------------------------------------------------------------ - - this module provides procedures for buffered reading of - standard input; - as usual, all identifiers of a module have the same prefix, - in this case 'read'; one underline will follow in case of a - global accessible object, two underlines in case of objects - which are not meant to be accessed from outside this module; - the sections of private and public definitions are separated - by a horizontal line: ---- */ - -#define read_PREFETCH ((32+3)*1024*1024) -/* number of bytes which will be available in the buffer after - every call of read_input(); - (important for reading .pbf files: - size must be greater than pb__blockM) */ -#define read__bufM (read_PREFETCH*5) /* length of the buffer; */ -typedef struct { /* members may not be accessed from external */ - int fd; /* file descriptor */ - bool eof; /* we are at the end of input file */ - byte* bufp; /* pointer in buf[] */ - byte* bufe; /* pointer to the end of valid input in buf[] */ - int64_t read__counter; - /* byte counter to get the read position in input file; */ - uint64_t bufferstart; - /* dummy variable which marks the start of the read buffer - concatenated with this instance of read info structure; */ - } read_info_t; - -/*------------------------------------------------------------*/ - -static read_info_t* read_infop= NULL; -/* presently used read info structure, i.e. file handle */ -#define read__buf ((byte*)&read_infop->bufferstart) -/* start address of the file's input buffer */ -static byte* read_bufp= NULL; /* may be incremented by external */ -/* up to the number of read_PREFETCH bytes before read_input() is - called again; */ -static byte* read_bufe= NULL; /* may not be changed from external */ - -static int read_open(const char* filename) { - /* open an input file; - filename[]: path and name of input file; - ==NULL: standard input; - return: 0: ok; !=0: error; - read_infop: handle of the file; - note that you should close every opened file with read_close() - before the program ends; - - save status of presently processed input file (if any) */ - if(read_infop!=NULL) { - read_infop->bufp= read_bufp; - read_infop->bufp= read_bufe; - } - - /* get memory space for file information and input buffer */ - read_infop= (read_info_t*)malloc(sizeof(read_info_t)+read__bufM); - if(read_infop==NULL) { - PERRv("could not get %i bytes of memory.",read__bufM) -return 1; - } - - /* initialize read info structure */ - read_infop->fd= 0; /* (default) standard input */ - read_infop->eof= false; /* we are at the end of input file */ - read_infop->bufp= read_infop->bufe= read__buf; /* pointer in buf[] */ - /* pointer to the end of valid input in buf[] */ - read_infop->read__counter= 0; - - /* set modul-global variables which are associated with this file */ - read_bufp= read_infop->bufp; - read_bufe= read_infop->bufe; - - /* open the file */ - if(loglevel>=2) - fprintf(stderr,"Read-opening: %s", - filename==NULL? "stdin": filename); - if(filename==NULL) /* stdin shall be opened */ - read_infop->fd= 0; - else if(filename!=NULL) { /* a real file shall be opened */ - read_infop->fd= open(filename,O_RDONLY|O_BINARY); - if(read_infop->fd<0) { - if(loglevel>=2) - fprintf(stderr," -> failed\n"); - PERRv("could not open input file: %.80s\n", - filename==NULL? "standard input": filename) - free(read_infop); read_infop= NULL; - read_bufp= read_bufe= NULL; -return 1; - } - } /* end a real file shall be opened */ - if(loglevel>=2) - fprintf(stderr," -> FD %i\n",read_infop->fd); -return 0; -} /* end read_open() */ - -static void read_close() { - /* close an opened file; - read_infop: handle of the file which is to close; */ - int fd; - - if(read_infop==NULL) /* handle not valid; */ -return; - fd= read_infop->fd; - if(loglevel>=1) { /* verbose */ - fprintf(stderr,"osm2pgsql: Number of bytes read: %"PRIu64"\n", - read_infop->read__counter); - } - if(loglevel>=2) { - fprintf(stderr,"Read-closing FD: %i\n",fd); - } - if(fd>0) /* not standard input */ - close(fd); - free(read_infop); read_infop= NULL; - read_bufp= read_bufe= NULL; -} /* end read_close() */ - -static inline bool read_input() { - /* read data from standard input file, use an internal buffer; - make data available at read_bufp; - read_open() must have been called before calling this procedure; - return: there are no (more) bytes to read; - read_bufp: start of next bytes available; - may be incremented by the caller, up to read_bufe; - read_bufe: end of bytes in buffer; - must not be changed by the caller; - after having called this procedure, the caller may rely on - having available at least read_PREFETCH bytes at address - read_bufp - with one exception: if there are not enough bytes - left to read from standard input, every byte after the end of - the reminding part of the file in the buffer will be set to - 0x00 - up to read_bufp+read_PREFETCH; */ - int l,r; - - if(read_bufp+read_PREFETCH>=read_bufe) { /* read buffer is too low */ - if(!read_infop->eof) { /* still bytes in the file */ - if(read_bufe>read_bufp) { /* bytes remaining in buffer */ - memmove(read__buf,read_bufp,read_bufe-read_bufp); - /* move remaining bytes to start of buffer */ - read_bufe= read__buf+(read_bufe-read_bufp); - /* protect the remaining bytes at buffer start */ - } - else /* no remaining bytes in buffer */ - read_bufe= read__buf; /* no bytes remaining to protect */ - /* add read bytes to debug counter */ - read_bufp= read__buf; - do { /* while buffer has not been filled */ - l= (read__buf+read__bufM)-read_bufe-4; - /* number of bytes to read */ - r= read(read_infop->fd,read_bufe,l); - if(r<=0) { /* no more bytes in the file */ - read_infop->eof= true; - /* memorize that there we are at end of file */ - l= (read__buf+read__bufM)-read_bufe; - /* reminding space in buffer */ - if(l>read_PREFETCH) l= read_PREFETCH; - memset(read_bufe,0,l); - /* set reminding space up to prefetch bytes in buffer to 0 */ - break; - } - read_infop->read__counter+= r; - read_bufe+= r; /* set new mark for end of data */ - read_bufe[0]= 0; read_bufe[1]= 0; /* set 4 null-terminators */ - read_bufe[2]= 0; read_bufe[3]= 0; - } while(reof && read_bufp>=read_bufe; -} /* end read__input() */ - - -/*------------------------------------------------------------ - end Module read_ OSM file read module - ------------------------------------------------------------ */ - - - -/*------------------------------------------------------------ - Module str_ string read module - ------------------------------------------------------------ - - this module provides procedures for conversions from - strings which have been stored in data stream objects to - c-formatted strings; - as usual, all identifiers of a module have the same prefix, - in this case 'str'; one underline will follow in case of a - global accessible object, two underlines in case of objects - which are not meant to be accessed from outside this module; - the sections of private and public definitions are separated - by a horizontal line: ---- */ - -#define str__tabM (15000+4000) -/* +4000 because it might happen that an object has a lot of - key/val pairs or refroles which are not stored already; */ -#define str__tabstrM 250 /* must be < row size of str__rab[] */ -typedef struct str__info_struct { - /* members of this structure must not be accessed - from outside this module; */ - char tab[str__tabM][256]; - /* string table; see o5m documentation; - row length must be at least str__tabstrM+2; - each row contains a double string; each of the two strings - is terminated by a zero byte, the logical lengths must not - exceed str__tabstrM bytes in total; - the first str__tabM lines of this array are used as - input buffer for strings; */ - int tabi; /* index of last entered element in string table; */ - int tabn; /* number of valid strings in string table; */ - struct str__info_struct* prev; /* address of previous unit; */ - } str_info_t; -str_info_t* str__infop= NULL; - -static void str__end() { - /* clean-up this module; */ - str_info_t* p; - - while(str__infop!=NULL) { - p= str__infop->prev; - free(str__infop); - str__infop= p; - } -} /* end str__end() */ - -/*------------------------------------------------------------*/ - -static str_info_t* str_open() { - /* open an new string client unit; - this will allow us to process multiple o5m input files; - return: handle of the new unit; - ==NULL: error; - you do not need to care about closing the unit(s); */ - static bool firstrun= true; - str_info_t* prev; - - prev= str__infop; - str__infop= (str_info_t*)malloc(sizeof(str_info_t)); - if(str__infop==NULL) { - PERR("could not get memory for string buffer.") -return NULL; - } - str__infop->tabi= 0; - str__infop->tabn= 0; - str__infop->prev= prev; - if(firstrun) { - firstrun= false; - atexit(str__end); - } - return str__infop; -} /* end str_open() */ - - -static void inline str_reset() { - /* clear string table; - must be called before any other procedure of this module - and may be called every time the string processing shall - be restarted; */ - str__infop->tabi= str__infop->tabn= 0; -} /* end str_reset() */ - -static void str_read(byte** pp,char** s1p,char** s2p) { - /* read an o5m formatted string (pair), e.g. key/val, from - standard input buffer; - if got a string reference, resolve it, using an internal - string table; - no reference is used if the strings are longer than - 250 characters in total (252 including terminators); - pp: address of a buffer pointer; - this pointer will be incremented by the number of bytes - the converted protobuf element consumes; - s2p: ==NULL: read not a string pair but a single string; - return: - *s1p,*s2p: pointers to the strings which have been read; */ - char* p; - int len1,len2; - int ref; - - p= (char*)*pp; - if(*p==0) { /* string (pair) given directly */ - *s1p= ++p; - len1= strlen(p); - p+= len1+1; - if(s2p==NULL) { /* single string */ - /* p= strchr(p,0)+1; jump over second string (if any) */ - if(len1<=str__tabstrM) { - char* tmpcharp; - - /* single string short enough for string table */ - tmpcharp= stpcpy0(str__infop->tab[str__infop->tabi],*s1p); - tmpcharp[1]= 0; - /* add a second terminator, just in case someone will try - to read this single string as a string pair later; */ - if(++str__infop->tabi>=str__tabM) str__infop->tabi= 0; - if(str__infop->tabntabn++; - } /* end single string short enough for string table */ - } /* end single string */ - else { /* string pair */ - *s2p= p; - len2= strlen(p); - p+= len2+1; - if(len1+len2<=str__tabstrM) { - /* string pair short enough for string table */ - memcpy(str__infop->tab[str__infop->tabi],*s1p,len1+len2+2); - if(++str__infop->tabi>=str__tabM) str__infop->tabi= 0; - if(str__infop->tabntabn++; - } /* end string pair short enough for string table */ - } /* end string pair */ - *pp= (byte*)p; - } /* end string (pair) given directly */ - else { /* string (pair) given by reference */ - ref= pbf_uint32(pp); - if(ref>str__infop->tabn) { /* string reference invalid */ - WARNv("invalid .o5m string reference: %i->%i", - str__infop->tabn,ref) - *s1p= "(invalid)"; - if(s2p!=NULL) /* caller wants a string pair */ - *s2p= "(invalid)"; - } /* end string reference invalid */ - else { /* string reference valid */ - ref= str__infop->tabi-ref; - if(ref<0) ref+= str__tabM; - *s1p= str__infop->tab[ref]; - if(s2p!=NULL) /* caller wants a string pair */ - *s2p= strchr(str__infop->tab[ref],0)+1; - } /* end string reference valid */ - } /* end string (pair) given by reference */ -} /* end str_read() */ - -/*------------------------------------------------------------ - end Module str_ string read module - ------------------------------------------------------------ */ - - - -int streamFileO5m(char *filename,int sanitize,struct osmdata_t *osmdata) { - /* open and parse an .o5m file; */ - /* return: ==0: ok; !=0: error; */ - int otype; /* type of currently processed object; */ - /* 0: node; 1: way; 2: relation; */ - uint32_t hisver; - int64_t histime; - int64_t hiscset; - uint32_t hisuid; - char* hisuser; - str_info_t* str; /* string unit handle (if o5m format) */ - bool endoffile; - int64_t o5id; /* for o5m delta coding */ - int32_t o5lon,o5lat; /* for o5m delta coding */ - int64_t o5histime; /* for o5m delta coding */ - int64_t o5hiscset; /* for o5m delta coding */ - int64_t o5rid[3]; /* for o5m delta coding */ - byte* bufp; /* pointer in read buffer */ -#define bufsp ((char*)bufp) /* for signed char */ - byte* bufe; /* pointer in read buffer, end of object */ - char c; /* latest character which has been read */ - byte b; /* latest byte which has been read */ - int l; - byte* bp; - - /* procedure initializations */ - str= str_open(); - /* call some initialization of string read module */ - str_reset(); - o5id= 0; - o5lat= o5lon= 0; - o5hiscset= 0; - o5histime= 0; - o5rid[0]= o5rid[1]= o5rid[2]= 0; - - /* open the input file */ - if(read_open(filename)!=0) { - fprintf(stderr,"Unable to open %s\n",filename); -return 1; - } - endoffile= false; - - /* determine file type */ { - char* p; - - read_input(); - if(*read_bufp!=0xff) { /* cannot be an .o5m file, nor an .o5c file */ - PERR("File format neither .o5m nor .o5c") -return 1; - } - p= strchr(filename,0)-4; /* get end of filename */ - if(memcmp(read_bufp,"\xff\xe0\0x04""o5m2",7)==0) - osmdata->filetype= FILETYPE_OSM; - else if(memcmp(read_bufp,"\xff\xe0\0x04""o5c2",7)==0) - osmdata->filetype= FILETYPE_OSMCHANGE; - else if(p>=filename && strcmp(p,".o5m")==0) - osmdata->filetype= FILETYPE_OSM; - else if(p>=filename && (strcmp(p,".o5c")==0 || strcmp(p,".o5h")==0)) - osmdata->filetype= FILETYPE_OSMCHANGE; - else { - WARN("File type not specified. Assuming .o5m") - osmdata->filetype= FILETYPE_OSM; - } - if(osmdata->filetype==FILETYPE_OSM) - PINFO("Processing .o5m file (not a change file).") - else - PINFO("Processing .o5c change file.") - } - - /* process the input file */ - for(;;) { /* read input file */ - - /* get next object */ - read_input(); - bufp= read_bufp; - b= *bufp; c= (char)b; - - /* care about file end */ - if(read_bufp>=read_bufe) /* at end of input file; */ - break; - - if(endoffile) { /* after logical end of file */ - fprintf(stderr,"osm2pgsql Warning: unexpected contents " - "after logical end of file.\n"); - break; - } - - /* care about header and unknown objects */ - if(b<0x10 || b>0x12) { /* not a regular dataset id */ - if(b>=0xf0) { /* single byte dataset */ - if(b==0xff) { /* file start, resp. o5m reset */ - str_reset(); - o5id= 0; - o5lat= o5lon= 0; - o5hiscset= 0; - o5histime= 0; - o5rid[0]= o5rid[1]= o5rid[2]= 0; - } - else if(b==0xfe) - endoffile= true; - else - WARNv("unknown .o5m short dataset id: 0x%02x\n",b) - read_bufp++; - continue; - } /* end single byte dataset */ - else { /* unknown multibyte dataset */ - if(b!=0xe0 && b!=0xdc) - WARNv("unknown .o5m dataset id: 0x%02x\n",b) - read_bufp++; - l= pbf_uint32(&read_bufp); /* jump over this dataset */ - read_bufp+= l; /* jump over this dataset */ - continue; - } /* end unknown multibyte dataset */ - } /* end not a regular dataset id */ - otype= b&3; - - /* object initialization */ - hisver= 0; - histime= 0; - hiscset= 0; - hisuid= 0; - hisuser= ""; - osmdata->nd_count= 0; - osmdata->member_count= 0; - - /* read object id */ - bufp++; - l= pbf_uint32(&bufp); - read_bufp= bufe= bufp+l; - osmdata->osm_id= o5id+= pbf_sint64(&bufp); - - /* do statistics on object id */ - switch(otype) { - case 0: /* node */ - if(osmdata->osm_id>osmdata->max_node) - osmdata->max_node= osmdata->osm_id; - if (osmdata->count_node == 0) { - time(&osmdata->start_node); - } - osmdata->count_node++; - if(osmdata->count_node%10000==0) printStatus(osmdata); - break; - case 1: /* way */ - if(osmdata->osm_id>osmdata->max_way) - osmdata->max_way= osmdata->osm_id; - if (osmdata->count_way == 0) { - time(&osmdata->start_way); - } - osmdata->count_way++; - if(osmdata->count_way%1000==0) printStatus(osmdata); - break; - case 2: /* relation */ - if(osmdata->osm_id>osmdata->max_rel) - osmdata->max_rel= osmdata->osm_id; - if (osmdata->count_rel == 0) { - time(&osmdata->start_rel); - } - osmdata->count_rel++; - if(osmdata->count_rel%10==0) printStatus(osmdata); - break; - default: ; - } - - /* read history */ { - char tmpstr[50]; - char* sp; - - hisver= pbf_uint32(&bufp); - uint32toa(hisver,tmpstr); - addItem(&(osmdata->tags),"osm_version",tmpstr,0); - if(hisver!=0) { /* history information available */ - histime= o5histime+= pbf_sint64(&bufp); - createtimestamp(histime,tmpstr); - addItem(&(osmdata->tags),"osm_timestamp",tmpstr, 0); - if(histime!=0) { - hiscset= o5hiscset+= pbf_sint32(&bufp); /* (not used) */ - str_read(&bufp,&sp,&hisuser); - hisuid= pbf_uint64((byte**)&sp); - uint32toa(hisuid,tmpstr); - addItem(&(osmdata->tags),"osm_uid",tmpstr,0); - addItem(&(osmdata->tags),"osm_user",hisuser,0); - } - } /* end history information available */ - } /* end read history */ - - /* perform action */ - if(bufp>=bufe) { - /* just the id and history, i.e. this is a delete request */ - osmdata->action= ACTION_DELETE; - switch(otype) { - case 0: /* node */ - osmdata->out->node_delete(osmdata->osm_id); - break; - case 1: /* way */ - osmdata->out->way_delete(osmdata->osm_id); - break; - case 2: /* relation */ - osmdata->out->relation_delete(osmdata->osm_id); - break; - default: ; - } - resetList(&(osmdata->tags)); - continue; /* end processing for this object */ - } /* end delete request */ - else { /* not a delete request */ - - /* determine action */ - if(osmdata->filetype==FILETYPE_OSMCHANGE && hisver>1) - osmdata->action= ACTION_MODIFY; - else - osmdata->action= ACTION_CREATE; - - /* read coordinates (for nodes only) */ - if(otype==0) { /* node */ - /* read node body */ - osmdata->node_lon= (double)(o5lon+= pbf_sint32(&bufp))/10000000; - osmdata->node_lat= (double)(o5lat+= pbf_sint32(&bufp))/10000000; - if(!node_wanted(osmdata,osmdata->node_lat,osmdata->node_lon)) { - resetList(&(osmdata->tags)); - continue; - } - reproject(&(osmdata->node_lat),&(osmdata->node_lon)); - } /* end node */ - - /* read noderefs (for ways only) */ - if(otype==1) { /* way */ - l= pbf_uint32(&bufp); - bp= bufp+l; - if(bp>bufe) bp= bufe; /* (format error) */ - while(bufpnds[osmdata->nd_count++]= o5rid[0]+= pbf_sint64(&bufp); - if(osmdata->nd_count>=osmdata->nd_max) - realloc_nodes(osmdata); - } /* end for all noderefs of this way */ - } /* end way */ - - /* read refs (for relations only) */ - else if(otype==2) { /* relation */ - int64_t ri; /* temporary, refid */ - int rt; /* temporary, reftype */ - char* rr; /* temporary, refrole */ - - l= pbf_uint32(&bufp); - bp= bufp+l; - if(bp>bufe) bp= bufe; /* (format error) */ - while(bufpmembers[osmdata->member_count].type= OSMTYPE_NODE; - break; - case 1: /* way */ - osmdata->members[osmdata->member_count].type= OSMTYPE_WAY; - break; - case 2: /* relation */ - osmdata->members[osmdata->member_count].type= OSMTYPE_RELATION; - break; - } - osmdata->members[osmdata->member_count].id= o5rid[rt]+= ri; - osmdata->members[osmdata->member_count].role= rr; - osmdata->member_count++; - if(osmdata->member_count>=osmdata->member_max) - realloc_members(osmdata); - } /* end for all references of this relation */ - } /* end relation */ - - /* read node key/val pairs */ - while(bufptags),k,v,0); - } - } /* end for all tags of this object */ - - /* write object into database */ - switch(otype) { - case 0: /* node */ - if(osmdata->action==ACTION_CREATE) - osmdata->out->node_add(osmdata->osm_id, - osmdata->node_lat,osmdata->node_lon,&(osmdata->tags)); - else /* ACTION_MODIFY */ - osmdata->out->node_modify(osmdata->osm_id, - osmdata->node_lat,osmdata->node_lon,&(osmdata->tags)); - break; - case 1: /* way */ - if(osmdata->action==ACTION_CREATE) - osmdata->out->way_add(osmdata->osm_id, - osmdata->nds,osmdata->nd_count,&(osmdata->tags)); - else /* ACTION_MODIFY */ - osmdata->out->way_modify(osmdata->osm_id, - osmdata->nds,osmdata->nd_count,&(osmdata->tags)); - break; - case 2: /* relation */ - if(osmdata->action==ACTION_CREATE) - osmdata->out->relation_add(osmdata->osm_id, - osmdata->members,osmdata->member_count,&(osmdata->tags)); - else /* ACTION_MODIFY */ - osmdata->out->relation_modify(osmdata->osm_id, - osmdata->members,osmdata->member_count,&(osmdata->tags)); - break; - default: ; - } - - /* reset temporary storage lists */ - resetList(&(osmdata->tags)); - - } /* end not a delete request */ - - } /* end read input file */ - - /* close the input file */ - printStatus(osmdata); - read_close(); - return 0; -} /* streamFileO5m() */ - diff -Nru osm2pgsql-0.82.0/parse-o5m.cpp osm2pgsql-0.88.1/parse-o5m.cpp --- osm2pgsql-0.82.0/parse-o5m.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse-o5m.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,962 @@ +/* +#----------------------------------------------------------------------------- +# osm2pgsql - converts planet.osm file into PostgreSQL +# compatible output suitable to be rendered by mapnik +#----------------------------------------------------------------------------- +# Original Python implementation by Artem Pavlenko +# Re-implementation by Jon Burgess, Copyright 2006 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +/* 2011-07-03 02:30 + Markus Weber */ + +// when __cplusplus is defined, we need to define this macro as well +// to get the print format specifiers in the inttypes.h header. +#define __STDC_FORMAT_MACROS +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +#include "parse-o5m.hpp" +#include "osmdata.hpp" +#include "osmtypes.hpp" +#include "reprojection.hpp" + +#define inline + +typedef uint8_t byte; +typedef unsigned int uint; +#define isdig(x) isdigit((unsigned char)(x)) +static int loglevel= 0; /* logging to stderr; */ +/* 0: no logging; 1: small logging; 2: normal logging; + 3: extended logging; */ +#define DP(f) fprintf(stderr,"- Debug: " #f "\n"); +#define DPv(f,...) fprintf(stderr,"- Debug: " #f "\n",__VA_ARGS__); +#ifdef _WIN32 +#define NL "\r\n" /* use CR/LF as new-line sequence */ + #define off_t off64_t + #define lseek lseek64 +#else +#define NL "\n" /* use LF as new-line sequence */ + #define O_BINARY 0 +#endif + +#define PERR(f) \ + fprintf(stderr,"osm2pgsql Error: " f "\n"); +/* print error message */ +#define PERRv(f,...) \ + fprintf(stderr,"osm2pgsql Error: " f "\n",__VA_ARGS__); +/* print error message with value(s) */ +#define WARN(f) { static int msgn= 3; if(--msgn>=0) \ + fprintf(stderr,"osm2pgsql Warning: " f "\n"); } +/* print a warning message, do it maximal 3 times */ +#define WARNv(f,...) { static int msgn= 3; if(--msgn>=0) \ + fprintf(stderr,"osm2pgsql Warning: " f "\n",__VA_ARGS__); } +/* print a warning message with value(s), do it maximal 3 times */ +#define PINFO(f) \ + fprintf(stderr,"osm2pgsql: " f "\n"); +/* print info message */ +#define ONAME(i) \ + (i==0? "node": i==1? "way": i==2? "relation": "unknown object") + +static inline char *stpcpy0(char *dest, const char *src) { + /* redefinition of C99's stpcpy() because it's missing in MinGW, + and declaration in Linux seems to be wrong; */ + while(*src!=0) + *dest++= *src++; + *dest= 0; + return dest; +} /* end stpcpy0() */ + +static inline char* uint32toa(uint32_t v,char* s) { + /* convert uint32_t integer into string; + v: long integer value to convert; + return: s; + s[]: digit string; */ + char* s1,*s2; + char c; + + s1= s; + if(v==0) + *s1++= '0'; + s2= s1; + while(v>0) + { *s2++= "0123456789"[v%10]; v/= 10; } + *s2--= 0; + while(s2>s1) + { c= *s1; *s1= *s2; *s2= c; s1++; s2--; } + return s; +} /* end uint32toa() */ + +static inline void createtimestamp(uint64_t v,char* sp) { + /* write a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z", + into a string; + v: value of the timestamp; + sp[21]: destination string; */ + time_t vtime; + struct tm tm; + int i; + + vtime= v; + #ifdef _WIN32 + memcpy(&tm,gmtime(&vtime),sizeof(tm)); + #else + gmtime_r(&vtime,&tm); + #endif + i= tm.tm_year+1900; + sp+= 3; *sp--= i%10+'0'; + i/=10; *sp--= i%10+'0'; + i/=10; *sp--= i%10+'0'; + i/=10; *sp= i%10+'0'; + sp+= 4; *sp++= '-'; + i= tm.tm_mon+1; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= '-'; + i= tm.tm_mday; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'T'; + i= tm.tm_hour; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; + i= tm.tm_min; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; + i= tm.tm_sec%60; + *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'Z'; *sp= 0; +} /* end createtimestamp() */ + + + +/*------------------------------------------------------------ + Module pbf_ protobuf conversions module + ------------------------------------------------------------ + + this module provides procedures for conversions from + protobuf formats to regular numbers; + as usual, all identifiers of a module have the same prefix, + in this case 'pbf'; one underline will follow in case of a + global accessible object, two underlines in case of objects + which are not meant to be accessed from outside this module; + the sections of private and public definitions are separated + by a horizontal line: ---- + many procedures have a parameter 'pp'; here, the address of + a buffer pointer is expected; this pointer will be incremented + by the number of bytes the converted protobuf element consumes; + + ------------------------------------------------------------ */ + +static inline uint32_t pbf_uint32(byte** pp) { + /* get the value of an unsigned integer; + pp: see module header; */ + byte* p; + uint32_t i; + uint32_t fac; + + p= *pp; + i= *p; + if((*p & 0x80)==0) { /* just one byte */ + (*pp)++; +return i; + } + i&= 0x7f; + fac= 0x80; + while(*++p & 0x80) { /* more byte(s) will follow */ + i+= (*p & 0x7f)*fac; + fac<<= 7; + } + i+= *p++ *fac; + *pp= p; + return i; +} /* end pbf_uint32() */ + +static inline int32_t pbf_sint32(byte** pp) { + /* get the value of an unsigned integer; + pp: see module header; */ + byte* p; + int32_t i; + int32_t fac; + int sig; + + p= *pp; + i= *p; + if((*p & 0x80)==0) { /* just one byte */ + (*pp)++; + if(i & 1) /* negative */ +return -1-(i>>1); + else +return i>>1; + } + sig= i & 1; + i= (i & 0x7e)>>1; + fac= 0x40; + while(*++p & 0x80) { /* more byte(s) will follow */ + i+= (*p & 0x7f)*fac; + fac<<= 7; + } + i+= *p++ *fac; + *pp= p; + if(sig) /* negative */ +return -1-i; + else +return i; +} /* end pbf_sint32() */ + +static inline uint64_t pbf_uint64(byte** pp) { + /* get the value of an unsigned integer; + pp: see module header; */ + byte* p; + uint64_t i; + uint64_t fac; + + p= *pp; + i= *p; + if((*p & 0x80)==0) { /* just one byte */ + (*pp)++; +return i; + } + i&= 0x7f; + fac= 0x80; + while(*++p & 0x80) { /* more byte(s) will follow */ + i+= (*p & 0x7f)*fac; + fac<<= 7; + } + i+= *p++ *fac; + *pp= p; + return i; +} /* end pbf_uint64() */ + +static inline int64_t pbf_sint64(byte** pp) { + /* get the value of a signed integer; + pp: see module header; */ + byte* p; + int64_t i; + int64_t fac; + int sig; + + p= *pp; + i= *p; + if((*p & 0x80)==0) { /* just one byte */ + (*pp)++; + if(i & 1) /* negative */ +return -1-(i>>1); + else +return i>>1; + } + sig= i & 1; + i= (i & 0x7e)>>1; + fac= 0x40; + while(*++p & 0x80) { /* more byte(s) will follow */ + i+= (*p & 0x7f)*fac; + fac<<= 7; + } + i+= *p++ *fac; + *pp= p; + if(sig) /* negative */ +return -1-i; + else +return i; +} /* end pbf_sint64() */ + +#if 0 /* not used at present */ +static inline void pbf_intjump(byte** pp) { + /* jump over a protobuf formatted integer; + pp: see module header; + we do not care about a possibly existing identifier, + therefore as the start address *pp the address of the + integer value is expected; */ + byte* p; + + p= *pp; + while(*p & 0x80) p++; p++; + *pp= p; +} /* end pbf_intjump() */ +#endif + +/*------------------------------------------------------------ + end Module pbf_ protobuf conversions module + ------------------------------------------------------------ */ + + + +/*------------------------------------------------------------ + Module read_ OSM file read module + ------------------------------------------------------------ + + this module provides procedures for buffered reading of + standard input; + as usual, all identifiers of a module have the same prefix, + in this case 'read'; one underline will follow in case of a + global accessible object, two underlines in case of objects + which are not meant to be accessed from outside this module; + the sections of private and public definitions are separated + by a horizontal line: ---- */ + +#define read_PREFETCH ((32+3)*1024*1024) +/* number of bytes which will be available in the buffer after + every call of read_input(); + (important for reading .pbf files: + size must be greater than pb__blockM) */ +#define read__bufM (read_PREFETCH*5) /* length of the buffer; */ +typedef struct { /* members may not be accessed from external */ + int fd; /* file descriptor */ + bool eof; /* we are at the end of input file */ + byte* bufp; /* pointer in buf[] */ + byte* bufe; /* pointer to the end of valid input in buf[] */ + int64_t read__counter; + /* byte counter to get the read position in input file; */ + uint64_t bufferstart; + /* dummy variable which marks the start of the read buffer + concatenated with this instance of read info structure; */ + } read_info_t; + +/*------------------------------------------------------------*/ + +static read_info_t* read_infop= NULL; +/* presently used read info structure, i.e. file handle */ +#define read__buf ((byte*)&read_infop->bufferstart) +/* start address of the file's input buffer */ +static byte* read_bufp= NULL; /* may be incremented by external */ +/* up to the number of read_PREFETCH bytes before read_input() is + called again; */ +static byte* read_bufe= NULL; /* may not be changed from external */ + +static int read_open(const char* filename) { + /* open an input file; + filename[]: path and name of input file; + ==NULL: standard input; + return: 0: ok; !=0: error; + read_infop: handle of the file; + note that you should close every opened file with read_close() + before the program ends; + + save status of presently processed input file (if any) */ + if(read_infop!=NULL) { + read_infop->bufp= read_bufp; + read_infop->bufp= read_bufe; + } + + /* get memory space for file information and input buffer */ + read_infop= (read_info_t*)malloc(sizeof(read_info_t)+read__bufM); + if(read_infop==NULL) { + PERRv("could not get %i bytes of memory.",read__bufM) +return 1; + } + + /* initialize read info structure */ + read_infop->fd= 0; /* (default) standard input */ + read_infop->eof= false; /* we are at the end of input file */ + read_infop->bufp= read_infop->bufe= read__buf; /* pointer in buf[] */ + /* pointer to the end of valid input in buf[] */ + read_infop->read__counter= 0; + + /* set modul-global variables which are associated with this file */ + read_bufp= read_infop->bufp; + read_bufe= read_infop->bufe; + + /* open the file */ + if(loglevel>=2) + fprintf(stderr,"Read-opening: %s", + filename==NULL? "stdin": filename); + if(filename==NULL) /* stdin shall be opened */ + read_infop->fd= 0; + else if(filename!=NULL) { /* a real file shall be opened */ + read_infop->fd= open(filename,O_RDONLY|O_BINARY); + if(read_infop->fd<0) { + if(loglevel>=2) + fprintf(stderr," -> failed\n"); + PERRv("could not open input file: %.80s\n", + filename==NULL? "standard input": filename) + free(read_infop); read_infop= NULL; + read_bufp= read_bufe= NULL; +return 1; + } + } /* end a real file shall be opened */ + if(loglevel>=2) + fprintf(stderr," -> FD %i\n",read_infop->fd); +return 0; +} /* end read_open() */ + +static void read_close() { + /* close an opened file; + read_infop: handle of the file which is to close; */ + int fd; + + if(read_infop==NULL) /* handle not valid; */ +return; + fd= read_infop->fd; + if(loglevel>=1) { /* verbose */ + fprintf(stderr,"osm2pgsql: Number of bytes read: %" PRIu64 "\n", + read_infop->read__counter); + } + if(loglevel>=2) { + fprintf(stderr,"Read-closing FD: %i\n",fd); + } + if(fd>0) /* not standard input */ + close(fd); + free(read_infop); read_infop= NULL; + read_bufp= read_bufe= NULL; +} /* end read_close() */ + +static inline bool read_input() { + /* read data from standard input file, use an internal buffer; + make data available at read_bufp; + read_open() must have been called before calling this procedure; + return: there are no (more) bytes to read; + read_bufp: start of next bytes available; + may be incremented by the caller, up to read_bufe; + read_bufe: end of bytes in buffer; + must not be changed by the caller; + after having called this procedure, the caller may rely on + having available at least read_PREFETCH bytes at address + read_bufp - with one exception: if there are not enough bytes + left to read from standard input, every byte after the end of + the reminding part of the file in the buffer will be set to + 0x00 - up to read_bufp+read_PREFETCH; */ + int l,r; + + if(read_bufp+read_PREFETCH>=read_bufe) { /* read buffer is too low */ + if(!read_infop->eof) { /* still bytes in the file */ + if(read_bufe>read_bufp) { /* bytes remaining in buffer */ + memmove(read__buf,read_bufp,read_bufe-read_bufp); + /* move remaining bytes to start of buffer */ + read_bufe= read__buf+(read_bufe-read_bufp); + /* protect the remaining bytes at buffer start */ + } + else /* no remaining bytes in buffer */ + read_bufe= read__buf; /* no bytes remaining to protect */ + /* add read bytes to debug counter */ + read_bufp= read__buf; + do { /* while buffer has not been filled */ + l= (read__buf+read__bufM)-read_bufe-4; + /* number of bytes to read */ + r= read(read_infop->fd,read_bufe,l); + if(r<=0) { /* no more bytes in the file */ + read_infop->eof= true; + /* memorize that there we are at end of file */ + l= (read__buf+read__bufM)-read_bufe; + /* reminding space in buffer */ + if(l>read_PREFETCH) l= read_PREFETCH; + memset(read_bufe,0,l); + /* set reminding space up to prefetch bytes in buffer to 0 */ + break; + } + read_infop->read__counter+= r; + read_bufe+= r; /* set new mark for end of data */ + read_bufe[0]= 0; read_bufe[1]= 0; /* set 4 null-terminators */ + read_bufe[2]= 0; read_bufe[3]= 0; + } while(reof && read_bufp>=read_bufe; +} /* end read__input() */ + + +/*------------------------------------------------------------ + end Module read_ OSM file read module + ------------------------------------------------------------ */ + + + +/*------------------------------------------------------------ + Module str_ string read module + ------------------------------------------------------------ + + this module provides procedures for conversions from + strings which have been stored in data stream objects to + c-formatted strings; + as usual, all identifiers of a module have the same prefix, + in this case 'str'; one underline will follow in case of a + global accessible object, two underlines in case of objects + which are not meant to be accessed from outside this module; + the sections of private and public definitions are separated + by a horizontal line: ---- */ + +#define str__tabM (15000+4000) +/* +4000 because it might happen that an object has a lot of + key/val pairs or refroles which are not stored already; */ +#define str__tabstrM 250 /* must be < row size of str__rab[] */ +typedef struct str__info_struct { + /* members of this structure must not be accessed + from outside this module; */ + char tab[str__tabM][256]; + /* string table; see o5m documentation; + row length must be at least str__tabstrM+2; + each row contains a double string; each of the two strings + is terminated by a zero byte, the logical lengths must not + exceed str__tabstrM bytes in total; + the first str__tabM lines of this array are used as + input buffer for strings; */ + int tabi; /* index of last entered element in string table; */ + int tabn; /* number of valid strings in string table; */ + struct str__info_struct* prev; /* address of previous unit; */ + } str_info_t; +str_info_t* str__infop= NULL; + +static void str__end() { + /* clean-up this module; */ + str_info_t* p; + + while(str__infop!=NULL) { + p= str__infop->prev; + free(str__infop); + str__infop= p; + } +} /* end str__end() */ + +/*------------------------------------------------------------*/ + +static str_info_t* str_open() { + /* open an new string client unit; + this will allow us to process multiple o5m input files; + return: handle of the new unit; + ==NULL: error; + you do not need to care about closing the unit(s); */ + static bool firstrun= true; + str_info_t* prev; + + prev= str__infop; + str__infop= (str_info_t*)malloc(sizeof(str_info_t)); + if(str__infop==NULL) { + PERR("could not get memory for string buffer.") +return NULL; + } + str__infop->tabi= 0; + str__infop->tabn= 0; + str__infop->prev= prev; + if(firstrun) { + firstrun= false; + atexit(str__end); + } + return str__infop; +} /* end str_open() */ + + +static void inline str_reset() { + /* clear string table; + must be called before any other procedure of this module + and may be called every time the string processing shall + be restarted; */ + str__infop->tabi= str__infop->tabn= 0; +} /* end str_reset() */ + +static void str_read(byte** pp,char** s1p,char** s2p) { + /* read an o5m formatted string (pair), e.g. key/val, from + standard input buffer; + if got a string reference, resolve it, using an internal + string table; + no reference is used if the strings are longer than + 250 characters in total (252 including terminators); + pp: address of a buffer pointer; + this pointer will be incremented by the number of bytes + the converted protobuf element consumes; + s2p: ==NULL: read not a string pair but a single string; + return: + *s1p,*s2p: pointers to the strings which have been read; */ + char* p; + int len1,len2; + int ref; + + p= (char*)*pp; + if(*p==0) { /* string (pair) given directly */ + *s1p= ++p; + len1= strlen(p); + p+= len1+1; + if(s2p==NULL) { /* single string */ + /* p= strchr(p,0)+1; jump over second string (if any) */ + if(len1<=str__tabstrM) { + char* tmpcharp; + + /* single string short enough for string table */ + tmpcharp= stpcpy0(str__infop->tab[str__infop->tabi],*s1p); + tmpcharp[1]= 0; + /* add a second terminator, just in case someone will try + to read this single string as a string pair later; */ + if(++str__infop->tabi>=str__tabM) str__infop->tabi= 0; + if(str__infop->tabntabn++; + } /* end single string short enough for string table */ + } /* end single string */ + else { /* string pair */ + *s2p= p; + len2= strlen(p); + p+= len2+1; + if(len1+len2<=str__tabstrM) { + /* string pair short enough for string table */ + memcpy(str__infop->tab[str__infop->tabi],*s1p,len1+len2+2); + if(++str__infop->tabi>=str__tabM) str__infop->tabi= 0; + if(str__infop->tabntabn++; + } /* end string pair short enough for string table */ + } /* end string pair */ + *pp= (byte*)p; + } /* end string (pair) given directly */ + else { /* string (pair) given by reference */ + ref= pbf_uint32(pp); + if(ref>str__infop->tabn) { /* string reference invalid */ + WARNv("invalid .o5m string reference: %i->%i", + str__infop->tabn,ref); + *s1p= strdup("(invalid)"); + if(s2p!=NULL) /* caller wants a string pair */ + *s2p= strdup("(invalid)"); + } /* end string reference invalid */ + else { /* string reference valid */ + ref= str__infop->tabi-ref; + if(ref<0) ref+= str__tabM; + *s1p= str__infop->tab[ref]; + if(s2p!=NULL) /* caller wants a string pair */ + *s2p= strchr(str__infop->tab[ref],0)+1; + } /* end string reference valid */ + } /* end string (pair) given by reference */ +} /* end str_read() */ + +/*------------------------------------------------------------ + end Module str_ string read module + ------------------------------------------------------------ */ + +parse_o5m_t::parse_o5m_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon, const double minlat, const double maxlon, const double maxlat): + parse_t(extra_attributes_, bbox_, projection_, minlon, minlat, maxlon, maxlat) +{ + +} + +parse_o5m_t::~parse_o5m_t() +{ + +} + +int parse_o5m_t::streamFile(const char *filename, const int sanitize, osmdata_t *osmdata) { + /* open and parse an .o5m file; */ + /* return: ==0: ok; !=0: error; */ + int otype; /* type of currently processed object; */ + /* 0: node; 1: way; 2: relation; */ + uint32_t hisver; + int64_t histime; + uint32_t hisuid; + char* hisuser; + bool endoffile; + int64_t o5id; /* for o5m delta coding */ + int32_t o5lon,o5lat; /* for o5m delta coding */ + int64_t o5histime; /* for o5m delta coding */ + int64_t o5hiscset; /* for o5m delta coding */ + int64_t o5rid[3]; /* for o5m delta coding */ + byte* bufp; /* pointer in read buffer */ +#define bufsp ((char*)bufp) /* for signed char */ + byte* bufe; /* pointer in read buffer, end of object */ + byte b; /* latest byte which has been read */ + int l; + byte* bp; + + /* procedure initializations */ + str_open(); + /* call some initialization of string read module */ + str_reset(); + o5id= 0; + o5lat= o5lon= 0; + o5hiscset= 0; + o5histime= 0; + o5rid[0]= o5rid[1]= o5rid[2]= 0; + + /* open the input file */ + if(read_open(filename)!=0) { + fprintf(stderr,"Unable to open %s\n",filename); +return 1; + } + endoffile= false; + + /* determine file type */ { + const char* p = NULL; + + read_input(); + if(*read_bufp!=0xff) { /* cannot be an .o5m file, nor an .o5c file */ + PERR("File format neither .o5m nor .o5c") +return 1; + } + p= strchr(filename,0)-4; /* get end of filename */ + if(memcmp(read_bufp,"\xff\xe0\0x04""o5m2",7)==0) + filetype= FILETYPE_OSM; + else if(memcmp(read_bufp,"\xff\xe0\0x04""o5c2",7)==0) + filetype= FILETYPE_OSMCHANGE; + else if(p>=filename && strcmp(p,".o5m")==0) + filetype= FILETYPE_OSM; + else if(p>=filename && (strcmp(p,".o5c")==0 || strcmp(p,".o5h")==0)) + filetype= FILETYPE_OSMCHANGE; + else { + WARN("File type not specified. Assuming .o5m") + filetype= FILETYPE_OSM; + } + if(filetype==FILETYPE_OSM) + PINFO("Processing .o5m file (not a change file).") + else + PINFO("Processing .o5c change file.") + } + + /* process the input file */ + for(;;) { /* read input file */ + + /* get next object */ + read_input(); + bufp= read_bufp; + b= *bufp; + + /* care about file end */ + if(read_bufp>=read_bufe) /* at end of input file; */ + break; + + if(endoffile) { /* after logical end of file */ + fprintf(stderr,"osm2pgsql Warning: unexpected contents " + "after logical end of file.\n"); + break; + } + + /* care about header and unknown objects */ + if(b<0x10 || b>0x12) { /* not a regular dataset id */ + if(b>=0xf0) { /* single byte dataset */ + if(b==0xff) { /* file start, resp. o5m reset */ + str_reset(); + o5id= 0; + o5lat= o5lon= 0; + o5hiscset= 0; + o5histime= 0; + o5rid[0]= o5rid[1]= o5rid[2]= 0; + } + else if(b==0xfe) + endoffile= true; + else + WARNv("unknown .o5m short dataset id: 0x%02x\n",b) + read_bufp++; + continue; + } /* end single byte dataset */ + else { /* unknown multibyte dataset */ + if(b!=0xe0 && b!=0xdc) + WARNv("unknown .o5m dataset id: 0x%02x\n",b) + read_bufp++; + l= pbf_uint32(&read_bufp); /* jump over this dataset */ + read_bufp+= l; /* jump over this dataset */ + continue; + } /* end unknown multibyte dataset */ + } /* end not a regular dataset id */ + otype= b&3; + + /* object initialization */ + hisver= 0; + histime= 0; + hisuid= 0; + hisuser= NULL; + + nds.clear(); + members.clear(); + + /* read object id */ + bufp++; + l= pbf_uint32(&bufp); + read_bufp= bufe= bufp+l; + osm_id= o5id+= pbf_sint64(&bufp); + + /* do statistics on object id */ + switch(otype) { + case 0: /* node */ + if(osm_id>max_node) + max_node= osm_id; + if (count_node == 0) { + time(&start_node); + } + count_node++; + if(count_node%10000==0) print_status(); + break; + case 1: /* way */ + if(osm_id>max_way) + max_way= osm_id; + if (count_way == 0) { + time(&start_way); + } + count_way++; + if(count_way%1000==0) print_status(); + break; + case 2: /* relation */ + if(osm_id>max_rel) + max_rel= osm_id; + if (count_rel == 0) { + time(&start_rel); + } + count_rel++; + if(count_rel%10==0) print_status(); + break; + default: ; + } + + /* read history */ { + char tmpstr[50]; + char* sp; + + hisver= pbf_uint32(&bufp); + uint32toa(hisver,tmpstr); + tags.push_back(tag("osm_version",tmpstr)); + if(hisver!=0) { /* history information available */ + histime= o5histime+= pbf_sint64(&bufp); + createtimestamp(histime,tmpstr); + tags.push_back(tag("osm_timestamp",tmpstr)); + if(histime!=0) { + o5hiscset+= pbf_sint32(&bufp); /* (not used) */ + str_read(&bufp,&sp,&hisuser); + hisuid= pbf_uint64((byte**)&sp); + uint32toa(hisuid,tmpstr); + tags.push_back(tag("osm_uid",tmpstr)); + tags.push_back(tag("osm_user",hisuser)); + } + } /* end history information available */ + } /* end read history */ + + /* perform action */ + if(bufp>=bufe) { + /* just the id and history, i.e. this is a delete request */ + action= ACTION_DELETE; + switch(otype) { + case 0: /* node */ + osmdata->node_delete(osm_id); + break; + case 1: /* way */ + osmdata->way_delete(osm_id); + break; + case 2: /* relation */ + osmdata->relation_delete(osm_id); + break; + default: ; + } + tags.clear(); + continue; /* end processing for this object */ + } /* end delete request */ + else { /* not a delete request */ + + /* determine action */ + if(filetype==FILETYPE_OSMCHANGE && hisver>1) + action= ACTION_MODIFY; + else + action= ACTION_CREATE; + + /* read coordinates (for nodes only) */ + if(otype==0) { /* node */ + /* read node body */ + node_lon= (double)(o5lon+= pbf_sint32(&bufp))/10000000; + node_lat= (double)(o5lat+= pbf_sint32(&bufp))/10000000; + if(!node_wanted(node_lat,node_lon)) { + tags.clear(); + continue; + } + proj->reproject(&(node_lat),&(node_lon)); + } /* end node */ + + /* read noderefs (for ways only) */ + if(otype==1) { /* way */ + l= pbf_uint32(&bufp); + bp= bufp+l; + if(bp>bufe) bp= bufe; /* (format error) */ + while(bufpbufe) bp= bufe; /* (format error) */ + while(bufpnode_add(osm_id, + node_lat,node_lon,tags); + else /* ACTION_MODIFY */ + osmdata->node_modify(osm_id, + node_lat,node_lon,tags); + break; + case 1: /* way */ + if(action==ACTION_CREATE) + osmdata->way_add(osm_id, + nds,tags); + else /* ACTION_MODIFY */ + osmdata->way_modify(osm_id, + nds,tags); + break; + case 2: /* relation */ + if(action==ACTION_CREATE) + osmdata->relation_add(osm_id, + members,tags); + else /* ACTION_MODIFY */ + osmdata->relation_modify(osm_id, + members,tags); + break; + default: ; + } + + /* reset temporary storage lists */ + tags.clear(); + + } /* end not a delete request */ + + } /* end read input file */ + + /* close the input file */ + print_status(); + read_close(); + return 0; +} /* streamFileO5m() */ diff -Nru osm2pgsql-0.82.0/parse-o5m.h osm2pgsql-0.88.1/parse-o5m.h --- osm2pgsql-0.82.0/parse-o5m.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-o5m.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef PARSE_O5M_H -#define PARSE_O5M_h - -int streamFileO5m(char *filename,int sanitize,struct osmdata_t *osmdata); - -#endif - - - - diff -Nru osm2pgsql-0.82.0/parse-o5m.hpp osm2pgsql-0.88.1/parse-o5m.hpp --- osm2pgsql-0.82.0/parse-o5m.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse-o5m.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,44 @@ +/* +#----------------------------------------------------------------------------- +# osm2pgsql - converts planet.osm file into PostgreSQL +# compatible output suitable to be rendered by mapnik +#----------------------------------------------------------------------------- +# Original Python implementation by Artem Pavlenko +# Re-implementation by Jon Burgess, Copyright 2006 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#ifndef PARSE_O5M_H +#define PARSE_O5M_H + +#include "parse.hpp" +#include + +struct reprojection; + +class parse_o5m_t: public parse_t +{ +public: + parse_o5m_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon, const double minlat, const double maxlon, const double maxlat); + virtual ~parse_o5m_t(); + int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata); +protected: + parse_o5m_t(); +}; + +#endif diff -Nru osm2pgsql-0.82.0/parse-pbf.c osm2pgsql-0.88.1/parse-pbf.c --- osm2pgsql-0.82.0/parse-pbf.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-pbf.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,627 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#include -#include -#include -#include -#include - -#include -#include - -#include "osmtypes.h" -#include "output.h" -#include "reprojection.h" - -#include "fileformat.pb-c.h" -#include "osmformat.pb-c.h" - -#define UNUSED __attribute__ ((unused)) - -#define MAX_BLOCK_HEADER_SIZE 64*1024 -#define MAX_BLOB_SIZE 32*1024*1024 - -#define NANO_DEGREE .000000001 - -static uint32_t get_length(FILE *input) -{ - char buf[4]; - - if (1 != fread(buf, sizeof(buf), 1, input)) - return 0; - - return ntohl(*((size_t *)buf)); -} - -static void *realloc_or_free(void *p, size_t len) -{ - void *new = realloc(p, len); - - if (new == NULL) { - free(p); - } - - return new; -} - -static BlockHeader *read_header(FILE *input, void *buf) -{ - BlockHeader *header_msg; - size_t read, length = get_length(input); - - if (length < 1 || length > MAX_BLOCK_HEADER_SIZE) { - if (!feof(input)) { - fprintf(stderr, "Invalid blocksize %lu\n", (unsigned long)length); - } - return NULL; - } - - read = fread(buf, length, 1, input); - if (!read) { - perror("parse-pbf: error while reading header data"); - return NULL; - } - - header_msg = block_header__unpack (NULL, length, buf); - if (header_msg == NULL) { - fprintf(stderr, "Error unpacking BlockHeader message\n"); - return NULL; - } - - return header_msg; -} - -static Blob *read_blob(FILE *input, void *buf, int32_t length) -{ - Blob *blob_msg; - - if (length < 1 || length > MAX_BLOB_SIZE) { - fprintf(stderr, "Blob isn't present or exceeds minimum/maximum size\n"); - return NULL; - } - - if(1 != fread(buf, length, 1, input)) { - fprintf(stderr, "error reading blob content\n"); - return NULL; - } - - blob_msg = blob__unpack (NULL, length, buf); - if (blob_msg == NULL) { - fprintf(stderr, "Error unpacking Blob message\n"); - return NULL; - } - - return blob_msg; -} - -static size_t uncompress_blob(Blob *bmsg, void *buf, int32_t max_size) -{ - if (bmsg->raw_size > max_size) { - fprintf(stderr, "blob raw size too large\n"); - return 0; - } - - if (bmsg->has_raw) { - memcpy(buf, bmsg->raw.data, bmsg->raw.len); - return bmsg->raw.len; - } else if (bmsg->has_zlib_data) { - int ret; - z_stream strm; - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = bmsg->zlib_data.len; - strm.next_in = bmsg->zlib_data.data; - strm.avail_out = bmsg->raw_size; - strm.next_out = buf; - - ret = inflateInit(&strm); - if (ret != Z_OK) { - fprintf(stderr, "Zlib init failed\n"); - return 0; - } - - ret = inflate(&strm, Z_NO_FLUSH); - - (void)inflateEnd(&strm); - - if (ret != Z_STREAM_END) { - fprintf(stderr, "Zlib compression failed (code %d, %s)\n", ret, strm.msg); - return 0; - } - - return bmsg->raw_size; - } else if (bmsg->has_bzip2_data) { - int ret; - bz_stream strm; - strm.bzalloc = NULL; - strm.bzfree = NULL; - strm.opaque = NULL; - strm.avail_in = bmsg->bzip2_data.len; - strm.next_in = (char *) bmsg->bzip2_data.data; - strm.avail_out = bmsg->raw_size; - strm.next_out = buf; - - ret = BZ2_bzDecompressInit(&strm, 0, 0); - if (ret != BZ_OK) { - fprintf(stderr, "Bzip2 init failed\n"); - return 0; - } - - (void)BZ2_bzDecompressEnd(&strm); - - if (ret != BZ_STREAM_END) { - fprintf(stderr, "Bzip2 compression failed\n"); - return 0; - } - - return bmsg->raw_size; - } else if (bmsg->has_lzma_data) { - fprintf(stderr, "Can't uncompress LZMA data\n"); - return 0; - } else { - fprintf(stderr, "We cannot handle the %d non-raw bytes yet...\n", bmsg->raw_size); - return 0; - } - - return 0; -} - -int addProtobufItem(struct keyval *head, ProtobufCBinaryData key, ProtobufCBinaryData val, int noDupe) -{ - char *keystr, *valstr; - int retval; - - keystr = calloc(key.len + 1, 1); - memcpy(keystr, key.data, key.len); - - /* drop certain keys (matching parse-xml2) */ - if ((strcmp(keystr, "created_by") == 0) || (strcmp(keystr, "source") == 0)) { - free(keystr); - return 0; - } - - valstr = calloc(val.len + 1, 1); - memcpy(valstr, val.data, val.len); - - retval = addItem(head, keystr, valstr, noDupe); - - free(keystr); - free(valstr); - - return retval; -} - -int addIntItem(struct keyval *head, const char *key, int val, int noDupe) -{ - char buf[100]; - - sprintf(buf, "%d", val); - return addItem(head, key, buf, noDupe); -} - -int addInfoItems(struct keyval *head, Info *info, StringTable *string_table) -{ - if (info->has_version) { - addIntItem(head, "osm_version", info->version, 0); - } - if (info->has_changeset) { - addIntItem(head, "osm_changeset", info->changeset, 0); - } - if (info->has_uid) { - addIntItem(head, "osm_uid", info->uid, 0); - } - if (info->has_user_sid) { - ProtobufCBinaryData user = string_table->s[info->user_sid]; - char *username; - - username = calloc(user.len + 1, 1); - memcpy(username, user.data, user.len); - - addItem(head, "osm_user", username, 0); - } - - /* TODO timestamp */ - - return 0; -} - -int processOsmHeader(void *data, size_t length) -{ - HeaderBlock *hmsg = header_block__unpack (NULL, length, data); - if (hmsg == NULL) { - fprintf(stderr, "Error unpacking HeaderBlock message\n"); - return 0; - } - - header_block__free_unpacked (hmsg, &protobuf_c_system_allocator); - - return 1; -} - -int processOsmDataNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity) -{ - unsigned node_id, key_id; - for (node_id = 0; node_id < group->n_nodes; node_id++) { - Node *node = group->nodes[node_id]; - double lat, lon; - - resetList(&(osmdata->tags)); - - if (node->info && osmdata->extra_attributes) { - addInfoItems(&(osmdata->tags), node->info, string_table); - } - - for (key_id = 0; key_id < node->n_keys; key_id++) { - addProtobufItem(&(osmdata->tags), - string_table->s[node->keys[key_id]], - string_table->s[node->vals[key_id]], - 0); - } - - lat = lat_offset + (node->lat * granularity); - lon = lon_offset + (node->lon * granularity); - if (node_wanted(osmdata, lat, lon)) { - reproject(&lat, &lon); - - osmdata->out->node_add(node->id, lat, lon, &(osmdata->tags)); - - if (node->id > osmdata->max_node) { - osmdata->max_node = node->id; - } - - if (osmdata->count_node == 0) { - time(&osmdata->start_node); - } - osmdata->count_node++; - if (osmdata->count_node%10000 == 0) - printStatus(osmdata); - } - } - - return 1; -} - -int processOsmDataDenseNodes(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity) -{ - unsigned node_id; - if (group->dense) { - unsigned l = 0; - osmid_t deltaid = 0; - long int deltalat = 0; - long int deltalon = 0; - unsigned long int deltatimestamp = 0; - unsigned long int deltachangeset = 0; - long int deltauid = 0; - unsigned long int deltauser_sid = 0; - double lat, lon; - - DenseNodes *dense = group->dense; - - for (node_id = 0; node_id < dense->n_id; node_id++) { - resetList(&(osmdata->tags)); - - deltaid += dense->id[node_id]; - deltalat += dense->lat[node_id]; - deltalon += dense->lon[node_id]; - - if (dense->denseinfo && osmdata->extra_attributes) { - DenseInfo *denseinfo = dense->denseinfo; - - deltatimestamp += denseinfo->timestamp[node_id]; - deltachangeset += denseinfo->changeset[node_id]; - deltauid += denseinfo->uid[node_id]; - deltauser_sid += denseinfo->user_sid[node_id]; - - addIntItem(&(osmdata->tags), "osm_version", denseinfo->version[node_id], 0); - addIntItem(&(osmdata->tags), "osm_changeset", deltachangeset, 0); - -#if 0 - /* TODO */ - if (deltauid != -1) { /* osmosis devs failed to read the specs */ - printuser(string_table->s[deltauser_sid]); - printnumericattribute("osm_uid", deltauid); - } - - printtimestamp("osm_timestamp", deltatimestamp); -#endif - } - - if (l < dense->n_keys_vals) { - while (dense->keys_vals[l] != 0 && l < dense->n_keys_vals) { - addProtobufItem(&(osmdata->tags), - string_table->s[dense->keys_vals[l]], - string_table->s[dense->keys_vals[l+1]], - 0); - - l += 2; - } - l += 1; - } - - lat = lat_offset + (deltalat * granularity); - lon = lon_offset + (deltalon * granularity); - if (node_wanted(osmdata, lat, lon)) { - reproject(&lat, &lon); - - osmdata->out->node_add(deltaid, lat, lon, &(osmdata->tags)); - - if (deltaid > osmdata->max_node) { - osmdata->max_node = deltaid; - } - - if (osmdata->count_node == 0) { - time(&osmdata->start_node); - } - osmdata->count_node++; - if (osmdata->count_node%10000 == 0) - printStatus(osmdata); - } - } - } - - return 1; -} - -int processOsmDataWays(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table) -{ - unsigned way_id, key_id, ref_id; - for (way_id = 0; way_id < group->n_ways; way_id++) { - Way *way = group->ways[way_id]; - osmid_t deltaref = 0; - - resetList(&(osmdata->tags)); - - if (way->info && osmdata->extra_attributes) { - addInfoItems(&(osmdata->tags), way->info, string_table); - } - - osmdata->nd_count = 0; - - for (ref_id = 0; ref_id < way->n_refs; ref_id++) { - deltaref += way->refs[ref_id]; - - osmdata->nds[osmdata->nd_count++] = deltaref; - - if( osmdata->nd_count >= osmdata->nd_max ) - realloc_nodes(osmdata); - } - - for (key_id = 0; key_id < way->n_keys; key_id++) { - addProtobufItem(&(osmdata->tags), - string_table->s[way->keys[key_id]], - string_table->s[way->vals[key_id]], - 0); - } - - osmdata->out->way_add(way->id, - osmdata->nds, - osmdata->nd_count, - &(osmdata->tags) ); - - if (way->id > osmdata->max_way) { - osmdata->max_way = way->id; - } - - if (osmdata->count_way == 0) { - time(&osmdata->start_way); - } - osmdata->count_way++; - if (osmdata->count_way%1000 == 0) - printStatus(osmdata); - } - - return 1; -} - -int processOsmDataRelations(struct osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table) -{ - unsigned rel_id, member_id, key_id; - for (rel_id = 0; rel_id < group->n_relations; rel_id++) { - Relation *relation = group->relations[rel_id]; - osmid_t deltamemids = 0; - - resetList(&(osmdata->tags)); - - osmdata->member_count = 0; - - if (relation->info && osmdata->extra_attributes) { - addInfoItems(&(osmdata->tags), relation->info, string_table); - } - - for (member_id = 0; member_id < relation->n_memids; member_id++) { - ProtobufCBinaryData role = string_table->s[relation->roles_sid[member_id]]; - char *rolestr; - - deltamemids += relation->memids[member_id]; - - osmdata->members[osmdata->member_count].id = deltamemids; - - rolestr = calloc(role.len + 1, 1); - memcpy(rolestr, role.data, role.len); - osmdata->members[osmdata->member_count].role = rolestr; - - switch (relation->types[member_id]) { - case RELATION__MEMBER_TYPE__NODE: - osmdata->members[osmdata->member_count].type = OSMTYPE_NODE; - break; - case RELATION__MEMBER_TYPE__WAY: - osmdata->members[osmdata->member_count].type = OSMTYPE_WAY; - break; - case RELATION__MEMBER_TYPE__RELATION: - osmdata->members[osmdata->member_count].type = OSMTYPE_RELATION; - break; - default: - fprintf(stderr, "Unsupported type: %u""\n", relation->types[member_id]); - return 0; - } - - osmdata->member_count++; - - if( osmdata->member_count >= osmdata->member_max ) { - realloc_members(osmdata); - } - } - - for (key_id = 0; key_id < relation->n_keys; key_id++) { - addProtobufItem(&(osmdata->tags), - string_table->s[relation->keys[key_id]], - string_table->s[relation->vals[key_id]], - 0); - } - - osmdata->out->relation_add(relation->id, - osmdata->members, - osmdata->member_count, - &(osmdata->tags)); - - for (member_id = 0; member_id < osmdata->member_count; member_id++) { - free(osmdata->members[member_id].role); - } - - if (relation->id > osmdata->max_rel) { - osmdata->max_rel = relation->id; - } - - if (osmdata->count_rel == 0) { - time(&osmdata->start_rel); - } - osmdata->count_rel++; - if (osmdata->count_rel%10 == 0) - printStatus(osmdata); - } - - return 1; -} - - -int processOsmData(struct osmdata_t *osmdata, void *data, size_t length) -{ - unsigned int j; - double lat_offset, lon_offset, granularity; - PrimitiveBlock *pmsg = primitive_block__unpack (NULL, length, data); - if (pmsg == NULL) { - fprintf(stderr, "Error unpacking PrimitiveBlock message\n"); - return 0; - } - - lat_offset = NANO_DEGREE * pmsg->lat_offset; - lon_offset = NANO_DEGREE * pmsg->lon_offset; - granularity = NANO_DEGREE * pmsg->granularity; - - for (j = 0; j < pmsg->n_primitivegroup; j++) { - PrimitiveGroup *group = pmsg->primitivegroup[j]; - StringTable *string_table = pmsg->stringtable; - - if (!processOsmDataNodes(osmdata, group, string_table, lat_offset, lon_offset, granularity)) return 0; - if (!processOsmDataDenseNodes(osmdata, group, string_table, lat_offset, lon_offset, granularity)) return 0; - if (!processOsmDataWays(osmdata, group, string_table)) return 0; - if (!processOsmDataRelations(osmdata, group, string_table)) return 0; - } - - primitive_block__free_unpacked (pmsg, &protobuf_c_system_allocator); - - return 1; -} - - - -int streamFilePbf(char *filename, int sanitize UNUSED, struct osmdata_t *osmdata) -{ - void *header = NULL; - void *blob = NULL; - char *data = NULL; - FILE *input = NULL; - BlockHeader *header_msg = NULL; - Blob *blob_msg = NULL; - size_t length; - int exit_status = EXIT_FAILURE; - - header = malloc(MAX_BLOCK_HEADER_SIZE); - if (!header) { - fprintf(stderr, "parse-pbf: out of memory allocating header buffer\n"); - goto err; - } - - blob = malloc(MAX_BLOB_SIZE); - if (!blob) { - fprintf(stderr, "parse-pbf: out of memory allocating blob buffer\n"); - goto err; - } - - data = malloc(MAX_BLOB_SIZE); - if (!data) { - fprintf(stderr, "parse-pbf: out of memory allocating data buffer\n"); - goto err; - } - - input = fopen(filename, "rb"); - if (!input) { - fprintf(stderr, "Unable to open %s\n", filename); - goto err; - } - - do { - header_msg = read_header(input, header); - if (header_msg == NULL) { - break; - } - - blob_msg = read_blob(input, blob, header_msg->datasize); - - length = uncompress_blob(blob_msg, data, MAX_BLOB_SIZE); - if (!length) { - goto err; - } - - if (strcmp(header_msg->type, "OSMHeader") == 0) { - if (!processOsmHeader(data, length)) { - goto err; - } - } else if (strcmp(header_msg->type, "OSMData") == 0) { - if (!processOsmData(osmdata, data, length)) { - goto err; - } - } - - blob__free_unpacked (blob_msg, &protobuf_c_system_allocator); - block_header__free_unpacked (header_msg, &protobuf_c_system_allocator); - } while (!feof(input)); - - if (!feof(input)) { - goto err; - } - - exit_status = EXIT_SUCCESS; - - err: - if (input) fclose(input); - - if (header) free(header); - if (blob) free(blob); - if (data) free(data); - - return exit_status; -} diff -Nru osm2pgsql-0.82.0/parse-pbf.cpp osm2pgsql-0.88.1/parse-pbf.cpp --- osm2pgsql-0.82.0/parse-pbf.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse-pbf.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,554 @@ +/* +#----------------------------------------------------------------------------- +# osm2pgsql - converts planet.osm file into PostgreSQL +# compatible output suitable to be rendered by mapnik +#----------------------------------------------------------------------------- +# Original Python implementation by Artem Pavlenko +# Re-implementation by Jon Burgess, Copyright 2006 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#include "config.h" + +#ifdef BUILD_READER_PBF + +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif +#include + +#include +#include + +#include "parse-pbf.hpp" +#include "osmdata.hpp" +#include "osmtypes.hpp" +#include "reprojection.hpp" + +#define MAX_BLOCK_HEADER_SIZE 64*1024 +#define MAX_BLOB_SIZE 32*1024*1024 + +#define NANO_DEGREE .000000001 + +static uint32_t get_length(FILE *input) +{ + uint32_t buf; + + if (1 != fread(reinterpret_cast(&buf), sizeof(buf), 1, input)) + return 0; + + return ntohl(buf); +} + +static BlockHeader *read_header(FILE *input, void *buf) +{ + BlockHeader *header_msg; + size_t read, length = get_length(input); + + if (length < 1 || length > MAX_BLOCK_HEADER_SIZE) { + if (!feof(input)) { + fprintf(stderr, "Invalid blocksize %lu\n", (unsigned long)length); + } + return NULL; + } + + read = fread(buf, length, 1, input); + if (!read) { + perror("parse-pbf: error while reading header data"); + return NULL; + } + + header_msg = block_header__unpack (NULL, length, (const uint8_t *)buf); + if (header_msg == NULL) { + fprintf(stderr, "Error unpacking BlockHeader message\n"); + return NULL; + } + + return header_msg; +} + +static Blob *read_blob(FILE *input, void *buf, int32_t length) +{ + Blob *blob_msg; + + if (length < 1 || length > MAX_BLOB_SIZE) { + fprintf(stderr, "Blob isn't present or exceeds minimum/maximum size\n"); + return NULL; + } + + if(1 != fread(buf, length, 1, input)) { + fprintf(stderr, "error reading blob content\n"); + return NULL; + } + + blob_msg = blob__unpack (NULL, length, (const uint8_t *)buf); + if (blob_msg == NULL) { + fprintf(stderr, "Error unpacking Blob message\n"); + return NULL; + } + + return blob_msg; +} + +static size_t uncompress_blob(Blob *bmsg, void *buf, int32_t max_size) +{ + if (bmsg->raw_size > max_size) { + fprintf(stderr, "blob raw size too large\n"); + return 0; + } + + if (bmsg->has_raw) { + memcpy(buf, bmsg->raw.data, bmsg->raw.len); + return bmsg->raw.len; + } else if (bmsg->has_zlib_data) { + int ret; + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = bmsg->zlib_data.len; + strm.next_in = bmsg->zlib_data.data; + strm.avail_out = bmsg->raw_size; + strm.next_out = (Bytef *)buf; + + ret = inflateInit(&strm); + if (ret != Z_OK) { + fprintf(stderr, "Zlib init failed\n"); + return 0; + } + + ret = inflate(&strm, Z_NO_FLUSH); + + (void)inflateEnd(&strm); + + if (ret != Z_STREAM_END) { + fprintf(stderr, "Zlib compression failed (code %d, %s)\n", ret, strm.msg); + return 0; + } + + return bmsg->raw_size; + } else if (bmsg->has_bzip2_data) { + fprintf(stderr, "Can't uncompress bz2 data\n"); + return 0; + } else if (bmsg->has_lzma_data) { + fprintf(stderr, "Can't uncompress LZMA data\n"); + return 0; + } else { + fprintf(stderr, "We cannot handle the %d non-raw bytes yet...\n", bmsg->raw_size); + return 0; + } + + return 0; +} + +void parse_pbf_t::addProtobufItem(ProtobufCBinaryData &key, + ProtobufCBinaryData &val) +{ + std::string keystr((const char *) key.data, key.len); + assert(keystr.find('\0') == std::string::npos); + + std::string valstr((const char *) val.data, val.len); + assert(valstr.find('\0') == std::string::npos); + + tags.push_back(tag(keystr, valstr)); +} + +void parse_pbf_t::addIntItem(const char *key, int val) +{ + char buf[100]; + sprintf(buf, "%d", val); + + tags.push_back(tag(key, buf)); +} + +void parse_pbf_t::addInfoItems(Info *info, StringTable *string_table) +{ + if (info->has_version) + addIntItem("osm_version", info->version); + + if (info->has_changeset) + addIntItem("osm_changeset", info->changeset); + + if (info->has_uid) + addIntItem("osm_uid", info->uid); + + if (info->has_user_sid) { + ProtobufCBinaryData user = string_table->s[info->user_sid]; + + tags.push_back(tag("osm_user", std::string((const char *) user.data, user.len))); + } + + /* TODO timestamp */ +} + +static int processOsmHeader(void *data, size_t length) +{ + HeaderBlock *hmsg = header_block__unpack (NULL, length, (const uint8_t *)data); + if (hmsg == NULL) { + fprintf(stderr, "Error unpacking HeaderBlock message\n"); + return 0; + } + + header_block__free_unpacked (hmsg, NULL); + + return 1; +} + +int parse_pbf_t::processOsmDataNodes(osmdata_t *osmdata, PrimitiveGroup *group, + StringTable *string_table, double lat_offset, + double lon_offset, double granularity) +{ + for (unsigned node_id = 0; node_id < group->n_nodes; node_id++) { + Node *node = group->nodes[node_id]; + + tags.clear(); + + if (node->info && extra_attributes) { + addInfoItems(node->info, string_table); + } + + for (unsigned key_id = 0; key_id < node->n_keys; key_id++) { + addProtobufItem(string_table->s[node->keys[key_id]], + string_table->s[node->vals[key_id]]); + } + + double lat = lat_offset + (node->lat * granularity); + double lon = lon_offset + (node->lon * granularity); + if (node_wanted(lat, lon)) { + proj->reproject(&lat, &lon); + + osmdata->node_add(node->id, lat, lon, tags); + + if (node->id > max_node) { + max_node = node->id; + } + + if (count_node == 0) { + time(&start_node); + } + count_node++; + if (count_node%10000 == 0) + print_status(); + } + } + + return 1; +} + +int parse_pbf_t::processOsmDataDenseNodes(osmdata_t *osmdata, PrimitiveGroup *group, + StringTable *string_table, + double lat_offset, double lon_offset, + double granularity) +{ + if (!group->dense) + return 1; + + unsigned l = 0; + osmid_t deltaid = 0; + long int deltalat = 0; + long int deltalon = 0; + unsigned long int deltatimestamp = 0; + unsigned long int deltachangeset = 0; + long int deltauid = 0; + unsigned long int deltauser_sid = 0; + + DenseNodes *dense = group->dense; + + for (unsigned node_id = 0; node_id < dense->n_id; node_id++) { + tags.clear(); + + deltaid += dense->id[node_id]; + deltalat += dense->lat[node_id]; + deltalon += dense->lon[node_id]; + + if (dense->denseinfo && extra_attributes) { + DenseInfo *denseinfo = dense->denseinfo; + + deltatimestamp += denseinfo->timestamp[node_id]; + deltachangeset += denseinfo->changeset[node_id]; + deltauid += denseinfo->uid[node_id]; + deltauser_sid += denseinfo->user_sid[node_id]; + + addIntItem("osm_version", denseinfo->version[node_id]); + addIntItem("osm_changeset", deltachangeset); + + if (deltauid != -1) { /* osmosis devs failed to read the specs */ + addIntItem("osm_uid", deltauid); + tags.push_back(tag("osm_user", + std::string((const char *) string_table->s[deltauser_sid].data, + string_table->s[deltauser_sid].len))); + } + } + + if (l < dense->n_keys_vals) { + while (dense->keys_vals[l] != 0 && l < dense->n_keys_vals) { + addProtobufItem(string_table->s[dense->keys_vals[l]], + string_table->s[dense->keys_vals[l+1]]); + + l += 2; + } + l += 1; + } + + double lat = lat_offset + (deltalat * granularity); + double lon = lon_offset + (deltalon * granularity); + if (node_wanted(lat, lon)) { + proj->reproject(&lat, &lon); + + osmdata->node_add(deltaid, lat, lon, tags); + + if (deltaid > max_node) { + max_node = deltaid; + } + + if (count_node == 0) { + time(&start_node); + } + count_node++; + if (count_node%10000 == 0) + print_status(); + } + } + + return 1; +} + +int parse_pbf_t::processOsmDataWays(osmdata_t *osmdata, PrimitiveGroup *group, + StringTable *string_table) +{ + for (unsigned way_id = 0; way_id < group->n_ways; way_id++) { + Way *way = group->ways[way_id]; + osmid_t deltaref = 0; + + tags.clear(); + + if (way->info && extra_attributes) { + addInfoItems(way->info, string_table); + } + + nds.clear(); + + for (unsigned ref_id = 0; ref_id < way->n_refs; ref_id++) { + deltaref += way->refs[ref_id]; + + nds.push_back(deltaref); + } + + for (unsigned key_id = 0; key_id < way->n_keys; key_id++) { + addProtobufItem(string_table->s[way->keys[key_id]], + string_table->s[way->vals[key_id]]); + } + + osmdata->way_add(way->id, nds, tags); + + if (way->id > max_way) { + max_way = way->id; + } + + if (count_way == 0) { + time(&start_way); + } + count_way++; + if (count_way%1000 == 0) + print_status(); + } + + return 1; +} + +int parse_pbf_t::processOsmDataRelations(osmdata_t *osmdata, PrimitiveGroup *group, + StringTable *string_table) +{ + for (unsigned rel_id = 0; rel_id < group->n_relations; rel_id++) { + Relation *relation = group->relations[rel_id]; + osmid_t deltamemids = 0; + + tags.clear(); + members.clear(); + + if (relation->info && extra_attributes) { + addInfoItems(relation->info, string_table); + } + + for (unsigned member_id = 0; member_id < relation->n_memids; member_id++) { + ProtobufCBinaryData role = string_table->s[relation->roles_sid[member_id]]; + + deltamemids += relation->memids[member_id]; + + OsmType type; + switch (relation->types[member_id]) { + case RELATION__MEMBER_TYPE__NODE: type = OSMTYPE_NODE; break; + case RELATION__MEMBER_TYPE__WAY: type = OSMTYPE_WAY; break; + case RELATION__MEMBER_TYPE__RELATION: type = OSMTYPE_RELATION; break; + default: + fprintf(stderr, "Unsupported type: %u""\n", relation->types[member_id]); + return 0; + } + + members.push_back(member(type, deltamemids, + std::string((const char *)role.data, role.len))); + } + + for (unsigned key_id = 0; key_id < relation->n_keys; key_id++) { + addProtobufItem(string_table->s[relation->keys[key_id]], + string_table->s[relation->vals[key_id]]); + } + + osmdata->relation_add(relation->id, members, tags); + + if (relation->id > max_rel) { + max_rel = relation->id; + } + + if (count_rel == 0) { + time(&start_rel); + } + count_rel++; + if (count_rel%10 == 0) + print_status(); + } + + return 1; +} + + +int parse_pbf_t::processOsmData(osmdata_t *osmdata, void *data, size_t length) +{ + unsigned int j; + double lat_offset, lon_offset, granularity; + PrimitiveBlock *pmsg = primitive_block__unpack (NULL, length, (const uint8_t *)data); + if (pmsg == NULL) { + fprintf(stderr, "Error unpacking PrimitiveBlock message\n"); + return 0; + } + + lat_offset = NANO_DEGREE * pmsg->lat_offset; + lon_offset = NANO_DEGREE * pmsg->lon_offset; + granularity = NANO_DEGREE * pmsg->granularity; + + for (j = 0; j < pmsg->n_primitivegroup; j++) { + PrimitiveGroup *group = pmsg->primitivegroup[j]; + StringTable *string_table = pmsg->stringtable; + + if (!processOsmDataNodes(osmdata, group, string_table, lat_offset, lon_offset, granularity)) return 0; + if (!processOsmDataDenseNodes(osmdata, group, string_table, lat_offset, lon_offset, granularity)) return 0; + if (!processOsmDataWays(osmdata, group, string_table)) return 0; + if (!processOsmDataRelations(osmdata, group, string_table)) return 0; + } + + primitive_block__free_unpacked (pmsg, NULL); + + return 1; +} + +parse_pbf_t::parse_pbf_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon, const double minlat, const double maxlon, const double maxlat): + parse_t(extra_attributes_, bbox_, projection_, minlon, minlat, maxlon, maxlat) +{ + +} + +parse_pbf_t::~parse_pbf_t() +{ + +} + +int parse_pbf_t::streamFile(const char *filename, const int, osmdata_t *osmdata) +{ + void *header = NULL; + void *blob = NULL; + char *data = NULL; + FILE *input = NULL; + BlockHeader *header_msg = NULL; + Blob *blob_msg = NULL; + size_t length; + int exit_status = EXIT_FAILURE; + + header = malloc(MAX_BLOCK_HEADER_SIZE); + if (!header) { + fprintf(stderr, "parse-pbf: out of memory allocating header buffer\n"); + goto err; + } + + blob = malloc(MAX_BLOB_SIZE); + if (!blob) { + fprintf(stderr, "parse-pbf: out of memory allocating blob buffer\n"); + goto err; + } + + data = (char *)malloc(MAX_BLOB_SIZE); + if (!data) { + fprintf(stderr, "parse-pbf: out of memory allocating data buffer\n"); + goto err; + } + + input = fopen(filename, "rb"); + if (!input) { + fprintf(stderr, "Unable to open %s\n", filename); + goto err; + } + + do { + header_msg = read_header(input, header); + if (header_msg == NULL) { + break; + } + + blob_msg = read_blob(input, blob, header_msg->datasize); + + length = uncompress_blob(blob_msg, data, MAX_BLOB_SIZE); + if (!length) { + goto err; + } + + if (strcmp(header_msg->type, "OSMHeader") == 0) { + if (!processOsmHeader(data, length)) { + goto err; + } + } else if (strcmp(header_msg->type, "OSMData") == 0) { + if (!processOsmData(osmdata, data, length)) { + goto err; + } + } + + blob__free_unpacked (blob_msg, NULL); + block_header__free_unpacked (header_msg, NULL); + } while (!feof(input)); + + if (!feof(input)) { + goto err; + } + + exit_status = EXIT_SUCCESS; + + err: + if (input) fclose(input); + + if (header) free(header); + if (blob) free(blob); + if (data) free(data); + + return exit_status; +} +#endif //BUILD_READER_PBF diff -Nru osm2pgsql-0.82.0/parse-pbf.h osm2pgsql-0.88.1/parse-pbf.h --- osm2pgsql-0.82.0/parse-pbf.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-pbf.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,30 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef PARSE_PBF_H -#define PARSE_PBF_h - -int streamFilePbf(char *filename, int sanitize, struct osmdata_t *osmdata); - -#endif diff -Nru osm2pgsql-0.82.0/parse-pbf.hpp osm2pgsql-0.88.1/parse-pbf.hpp --- osm2pgsql-0.82.0/parse-pbf.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse-pbf.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,67 @@ +/* +#----------------------------------------------------------------------------- +# osm2pgsql - converts planet.osm file into PostgreSQL +# compatible output suitable to be rendered by mapnik +#----------------------------------------------------------------------------- +# Original Python implementation by Artem Pavlenko +# Re-implementation by Jon Burgess, Copyright 2006 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#ifndef PARSE_PBF_H +#define PARSE_PBF_H + +#include "config.h" + +#ifdef BUILD_READER_PBF +extern "C" { +#include "fileformat.pb-c.h" +#include "osmformat.pb-c.h" +} + +#include "parse.hpp" + +#include +#include + +struct reprojection; +class osmdata_t; + +class parse_pbf_t: public parse_t +{ +public: + parse_pbf_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon, const double minlat, const double maxlon, const double maxlat); + virtual ~parse_pbf_t(); + virtual int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata); +protected: + parse_pbf_t(); + int processOsmDataNodes(osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity); + int processOsmDataDenseNodes(osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table, double lat_offset, double lon_offset, double granularity); + int processOsmDataWays(osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table); + int processOsmDataRelations(osmdata_t *osmdata, PrimitiveGroup *group, StringTable *string_table); + int processOsmData(osmdata_t *osmdata, void *data, size_t length); + +private: + void addProtobufItem(ProtobufCBinaryData &key, ProtobufCBinaryData &val); + void addIntItem(const char *key, int val); + void addInfoItems(Info *info, StringTable *string_table); +}; + +#endif //BUILD_READER_PBF + +#endif diff -Nru osm2pgsql-0.82.0/parse-primitive.c osm2pgsql-0.88.1/parse-primitive.c --- osm2pgsql-0.82.0/parse-primitive.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-primitive.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,486 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -/* - -This is a version of osm2pgsql without proper XML parsing -it should arrive at the same results as the normal osm2pgsql -and take an hour less to process the full planet file but -YMMV. This is just a proof of concept and should not be used -in a production environment. - - */ - -#define _GNU_SOURCE -#define UNUSED __attribute__ ((unused)) - -#include -#include -#include -#include -#include - -#include "osmtypes.h" -#include "sanitizer.h" -#include "reprojection.h" -#include "input.h" -#include "output.h" - - -char *extractAttribute(char **token, int tokens, char *attname) -{ - char buffer[256]; - int cl; - int i; - char *in; - char *out; - sprintf(buffer, "%s=\"", attname); - cl = strlen(buffer); - for (i=0; ifiletype == FILETYPE_OSMCHANGE || osmdata->filetype == FILETYPE_PLANETDIFF ) - return osmdata->action; - new_action = ACTION_NONE; - action = extractAttribute(token, tokens, "action"); - if( action == NULL ) - new_action = ACTION_CREATE; - else if( strcmp((char *)action, "modify") == 0 ) - new_action = ACTION_MODIFY; - else if( strcmp((char *)action, "delete") == 0 ) - new_action = ACTION_DELETE; - else - { - fprintf( stderr, "Unknown value for action: %s\n", (char*)action ); - exit_nicely(); - } - return new_action; -} - -static void StartElement(char *name, char *line, struct osmdata_t *osmdata) -{ - char *xid, *xlat, *xlon, *xk, *xv, *xrole, *xtype; - char *token[255]; - int tokens = 0; - int quote = 0; - char *i; - - if (osmdata->filetype == FILETYPE_NONE) - { - if (!strcmp(name, "?xml")) return; - if (!strcmp(name, "osm")) - { - osmdata->filetype = FILETYPE_OSM; - osmdata->action = ACTION_CREATE; - } - else if (!strcmp(name, "osmChange")) - { - osmdata->filetype = FILETYPE_OSMCHANGE; - osmdata->action = ACTION_NONE; - } - else if (!strcmp(name, "planetdiff")) - { - osmdata->filetype = FILETYPE_PLANETDIFF; - osmdata->action = ACTION_NONE; - } - else - { - fprintf( stderr, "Unknown XML document type: %s\n", name ); - exit_nicely(); - } - return; - } - - tokens=1; - token[0] = line; - for (i=line; *i; i++) - { - if (quote) - { - if (*i == '"') - { - quote = 0; - } - } - else - { - if (*i == '"') - { - quote = 1; - } - else if (isspace(*i)) - { - *i = 0; - token[tokens++] = i + 1; - } - } - } - - if (!strcmp(name, "node")) { - xid = extractAttribute(token, tokens, "id"); - xlon = extractAttribute(token, tokens, "lon"); - xlat = extractAttribute(token, tokens, "lat"); - assert(xid); assert(xlon); assert(xlat); - - osmdata->osm_id = strtoosmid((char *)xid, NULL, 10); - osmdata->node_lon = strtod((char *)xlon, NULL); - osmdata->node_lat = strtod((char *)xlat, NULL); - osmdata->action = ParseAction(token, tokens, osmdata); - - if (osmdata->osm_id > osmdata->max_node) - osmdata->max_node = osmdata->osm_id; - - if (osmdata->count_node == 0) { - time(&osmdata->start_node); - } - - osmdata->count_node++; - if (osmdata->count_node%10000 == 0) - printStatus(osmdata); - - } else if (!strcmp(name, "tag")) { - xk = extractAttribute(token, tokens, "k"); - assert(xk); - - /* 'created_by' and 'source' are common and not interesting to mapnik renderer */ - if (strcmp((char *)xk, "created_by") && strcmp((char *)xk, "source")) { - char *p; - xv = extractAttribute(token, tokens, "v"); - assert(xv); - while ((p = strchr(xk, ' '))) - *p = '_'; - - addItem(&(osmdata->tags), xk, (char *)xv, 0); - } - } else if (!strcmp(name, "way")) { - - xid = extractAttribute(token, tokens, "id"); - assert(xid); - osmdata->osm_id = strtoosmid((char *)xid, NULL, 10); - osmdata->action = ParseAction(token, tokens, osmdata); - - if (osmdata->osm_id > osmdata->max_way) - osmdata->max_way = osmdata->osm_id; - - if (osmdata->count_way == 0) { - time(&osmdata->start_way); - } - - osmdata->count_way++; - if (osmdata->count_way%1000 == 0) - printStatus(osmdata); - - osmdata->nd_count = 0; - } else if (!strcmp(name, "nd")) { - xid = extractAttribute(token, tokens, "ref"); - assert(xid); - - osmdata->nds[osmdata->nd_count++] = strtoosmid( (char *)xid, NULL, 10 ); - - if( osmdata->nd_count >= osmdata->nd_max ) - realloc_nodes(osmdata); - } else if (!strcmp(name, "relation")) { - xid = extractAttribute(token, tokens, "id"); - assert(xid); - osmdata->osm_id = strtoosmid((char *)xid, NULL, 10); - osmdata->action = ParseAction(token, tokens, osmdata); - - if (osmdata->osm_id > osmdata->max_rel) - osmdata->max_rel = osmdata->osm_id; - - if (osmdata->count_rel == 0) { - time(&osmdata->start_rel); - } - - osmdata->count_rel++; - if (osmdata->count_rel%10 == 0) - printStatus(osmdata); - - osmdata->member_count = 0; - } else if (!strcmp(name, "member")) { - xrole = extractAttribute(token, tokens, "role"); - assert(xrole); - - xtype = extractAttribute(token, tokens, "type"); - assert(xtype); - - xid = extractAttribute(token, tokens, "ref"); - assert(xid); - - osmdata->members[osmdata->member_count].id = strtoosmid( (char *)xid, NULL, 0 ); - osmdata->members[osmdata->member_count].role = strdup( (char *)xrole ); - - /* Currently we are only interested in 'way' members since these form polygons with holes */ - if (!strcmp(xtype, "way")) - osmdata->members[osmdata->member_count].type = OSMTYPE_WAY; - else if (!strcmp(xtype, "node")) - osmdata->members[osmdata->member_count].type = OSMTYPE_NODE; - else if (!strcmp(xtype, "relation")) - osmdata->members[osmdata->member_count].type = OSMTYPE_RELATION; - osmdata->member_count++; - - if( osmdata->member_count >= osmdata->member_max ) - realloc_members(osmdata); - } else if (!strcmp(name, "add") || - !strcmp(name, "create")) { - osmdata->action = ACTION_MODIFY; /* Turns all creates into modifies, makes it resiliant against inconsistant snapshots. */ - } else if (!strcmp(name, "modify")) { - osmdata->action = ACTION_MODIFY; - } else if (!strcmp(name, "delete")) { - osmdata->action = ACTION_DELETE; - } else if (!strcmp(name, "bound")) { - /* ignore */ - } else if (!strcmp(name, "bounds")) { - /* ignore */ - } else if (!strcmp(name, "changeset")) { - /* ignore */ - } else { - fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); - } - - /* Collect extra attribute information and add as tags */ - if (osmdata->extra_attributes && (!strcmp(name, "node") || - !strcmp(name, "way") || - !strcmp(name, "relation"))) - { - char *xtmp; - - xtmp = extractAttribute(token, tokens, "user"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_user", (char *)xtmp, 0); - } - - xtmp = extractAttribute(token, tokens, "uid"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_uid", (char *)xtmp, 0); - } - - xtmp = extractAttribute(token, tokens, "version"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_version", (char *)xtmp, 0); - } - - xtmp = extractAttribute(token, tokens, "timestamp"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_timestamp", (char *)xtmp, 0); - } - } -} - -static void EndElement(const char *name, struct osmdata_t *osmdata) -{ - if (!strcmp(name, "node")) { - if (node_wanted(osmdata, osmdata->node_lat, osmdata->node_lon)) { - reproject(&(osmdata->node_lat), &(osmdata->node_lon)); - if( osmdata->action == ACTION_CREATE ) - osmdata->out->node_add(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags)); - else if( osmdata->action == ACTION_MODIFY ) - osmdata->out->node_modify(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags)); - else if( osmdata->action == ACTION_DELETE ) - osmdata->out->node_delete(osmdata->osm_id); - else - { - fprintf( stderr, "Don't know action for node %" PRIdOSMID "\n", osmdata->osm_id ); - exit_nicely(); - } - } - resetList(&(osmdata->tags)); - } else if (!strcmp(name, "way")) { - if( osmdata->action == ACTION_CREATE ) - osmdata->out->way_add(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) ); - else if( osmdata->action == ACTION_MODIFY ) - osmdata->out->way_modify(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) ); - else if( osmdata->action == ACTION_DELETE ) - osmdata->out->way_delete(osmdata->osm_id); - else - { - fprintf( stderr, "Don't know action for way %" PRIdOSMID "\n", osmdata->osm_id ); - exit_nicely(); - } - resetList(&(osmdata->tags)); - } else if (!strcmp(name, "relation")) { - if( osmdata->action == ACTION_CREATE ) - osmdata->out->relation_add(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags)); - else if( osmdata->action == ACTION_MODIFY ) - osmdata->out->relation_modify(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags)); - else if( osmdata->action == ACTION_DELETE ) - osmdata->out->relation_delete(osmdata->osm_id); - else - { - fprintf( stderr, "Don't know action for relation %" PRIdOSMID "\n", osmdata->osm_id ); - exit_nicely(); - } - resetList(&(osmdata->tags)); - resetMembers(osmdata); - } else if (!strcmp(name, "tag")) { - /* ignore */ - } else if (!strcmp(name, "nd")) { - /* ignore */ - } else if (!strcmp(name, "member")) { - /* ignore */ - } else if (!strcmp(name, "osm")) { - printStatus(osmdata); - osmdata->filetype = FILETYPE_NONE; - } else if (!strcmp(name, "osmChange")) { - printStatus(osmdata); - osmdata->filetype = FILETYPE_NONE; - } else if (!strcmp(name, "planetdiff")) { - printStatus(osmdata); - osmdata->filetype = FILETYPE_NONE; - } else if (!strcmp(name, "bound")) { - /* ignore */ - } else if (!strcmp(name, "bounds")) { - /* ignore */ - } else if (!strcmp(name, "changeset")) { - /* ignore */ - resetList(&(osmdata->tags)); /* We may have accumulated some tags even if we ignored the changeset */ - } else if (!strcmp(name, "add")) { - osmdata->action = ACTION_NONE; - } else if (!strcmp(name, "create")) { - osmdata->action = ACTION_NONE; - } else if (!strcmp(name, "modify")) { - osmdata->action = ACTION_NONE; - } else if (!strcmp(name, "delete")) { - osmdata->action = ACTION_NONE; - } else { - fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); - } -} - -static void process(char *line, struct osmdata_t *osmdata) { - char *lt = strchr(line, '<'); - if (lt) - { - char *spc = strchr(lt+1, ' '); - char *gt = strchr(lt+1, '>'); - char *nx = spc; - if (*(lt+1) == '/') - { - *gt = 0; - EndElement(lt+2, osmdata); - } - else - { - int slash = 0; - if (gt != NULL) { - *gt-- = 0; - if (nx == NULL || gt < nx) nx = gt; - while(gt>lt) - { - if (*gt=='/') { slash=1; *gt=0; break; } - if (!isspace(*gt)) break; - gt--; - } - } - *nx++ = 0; - /* printf ("nx=%d, lt+1=#%s#\n", nx-lt,lt+1); */ - StartElement(lt+1, nx, osmdata); - if (slash) EndElement(lt+1, osmdata); - } - } -} - -int streamFilePrimitive(char *filename, int sanitize UNUSED, struct osmdata_t *osmdata) { - struct Input *i; - char buffer[65536]; - int bufsz = 0; - int offset = 0; - char *nl; - - i = inputOpen(filename); - - if (i != NULL) { - while(1) - { - bufsz = bufsz + readFile(i, buffer + bufsz, sizeof(buffer) - bufsz - 1); - buffer[bufsz] = 0; - nl = strchr(buffer, '\n'); - if (nl == 0) break; - *nl=0; - while (nl && nl < buffer + bufsz) - { - *nl = 0; - process(buffer + offset, osmdata); - offset = nl - buffer + 1; - nl = strchr(buffer + offset, '\n'); - } - memcpy(buffer, buffer + offset, bufsz - offset); - bufsz = bufsz - offset; - offset = 0; - } - } else { - fprintf(stderr, "Unable to open %s\n", filename); - return 1; - } - inputClose(i); - return 0; -} diff -Nru osm2pgsql-0.82.0/parse-primitive.h osm2pgsql-0.88.1/parse-primitive.h --- osm2pgsql-0.82.0/parse-primitive.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-primitive.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,30 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef PARSE_PRIMITIVE_H -#define PARSE_PRIMITIVE_H - -int streamFilePrimitive(char *filename, int sanitize, struct osmdata_t *osmdata); - -#endif diff -Nru osm2pgsql-0.82.0/parse-xml2.c osm2pgsql-0.88.1/parse-xml2.c --- osm2pgsql-0.82.0/parse-xml2.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-xml2.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,400 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#define _GNU_SOURCE - -#include -#include -#include -#include -#include - -#include -#include - -#include "osmtypes.h" -#include "sanitizer.h" -#include "reprojection.h" -#include "input.h" -#include "output.h" - -#include "parse-xml2.h" - - - -/* Parses the action="foo" tags in JOSM change files. Obvisouly not useful from osmChange files */ -static actions_t ParseAction( xmlTextReaderPtr reader, struct osmdata_t *osmdata ) -{ - actions_t new_action; - xmlChar *action; - if( osmdata->filetype == FILETYPE_OSMCHANGE || osmdata->filetype == FILETYPE_PLANETDIFF ) - return osmdata->action; - new_action = ACTION_NONE; - action = xmlTextReaderGetAttribute( reader, BAD_CAST "action" ); - if( action == NULL ) - new_action = ACTION_CREATE; - else if( strcmp((char *)action, "modify") == 0 ) - new_action = ACTION_MODIFY; - else if( strcmp((char *)action, "delete") == 0 ) - new_action = ACTION_DELETE; - else - { - fprintf( stderr, "Unknown value for action: %s\n", (char*)action ); - exit_nicely(); - } - return new_action; -} - -static void StartElement(xmlTextReaderPtr reader, const xmlChar *name, struct osmdata_t *osmdata) -{ - xmlChar *xid, *xlat, *xlon, *xk, *xv, *xrole, *xtype; - char *k; - - if (osmdata->filetype == FILETYPE_NONE) - { - if (xmlStrEqual(name, BAD_CAST "osm")) - { - osmdata->filetype = FILETYPE_OSM; - osmdata->action = ACTION_CREATE; - } - else if (xmlStrEqual(name, BAD_CAST "osmChange")) - { - osmdata->filetype = FILETYPE_OSMCHANGE; - osmdata->action = ACTION_NONE; - } - else if (xmlStrEqual(name, BAD_CAST "planetdiff")) - { - osmdata->filetype = FILETYPE_PLANETDIFF; - osmdata->action = ACTION_NONE; - } - else - { - fprintf( stderr, "Unknown XML document type: %s\n", name ); - exit_nicely(); - } - return; - } - - if (xmlStrEqual(name, BAD_CAST "node")) { - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); - xlon = xmlTextReaderGetAttribute(reader, BAD_CAST "lon"); - xlat = xmlTextReaderGetAttribute(reader, BAD_CAST "lat"); - assert(xid); assert(xlon); assert(xlat); - - osmdata->osm_id = strtoosmid((char *)xid, NULL, 10); - osmdata->node_lon = strtod((char *)xlon, NULL); - osmdata->node_lat = strtod((char *)xlat, NULL); - osmdata->action = ParseAction( reader , osmdata); - - if (osmdata->osm_id > osmdata->max_node) - osmdata->max_node = osmdata->osm_id; - - if (osmdata->count_node == 0) { - time(&osmdata->start_node); - } - osmdata->count_node++; - if (osmdata->count_node%10000 == 0) - printStatus(osmdata); - - xmlFree(xid); - xmlFree(xlon); - xmlFree(xlat); - } else if (xmlStrEqual(name, BAD_CAST "tag")) { - xk = xmlTextReaderGetAttribute(reader, BAD_CAST "k"); - assert(xk); - - /* 'created_by' and 'source' are common and not interesting to mapnik renderer */ - if (strcmp((char *)xk, "created_by") && strcmp((char *)xk, "source")) { - char *p; - xv = xmlTextReaderGetAttribute(reader, BAD_CAST "v"); - assert(xv); - k = (char *)xmlStrdup(xk); - while ((p = strchr(k, ' '))) - *p = '_'; - - addItem(&(osmdata->tags), k, (char *)xv, 0); - xmlFree(k); - xmlFree(xv); - } - xmlFree(xk); - } else if (xmlStrEqual(name, BAD_CAST "way")) { - - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); - assert(xid); - osmdata->osm_id = strtoosmid((char *)xid, NULL, 10); - osmdata->action = ParseAction( reader, osmdata ); - - if (osmdata->osm_id > osmdata->max_way) - osmdata->max_way = osmdata->osm_id; - - if (osmdata->count_way == 0) { - time(&osmdata->start_way); - } - osmdata->count_way++; - if (osmdata->count_way%1000 == 0) - printStatus(osmdata); - - osmdata->nd_count = 0; - xmlFree(xid); - - } else if (xmlStrEqual(name, BAD_CAST "nd")) { - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref"); - assert(xid); - - osmdata->nds[osmdata->nd_count++] = strtoosmid( (char *)xid, NULL, 10 ); - - if( osmdata->nd_count >= osmdata->nd_max ) - realloc_nodes(osmdata); - xmlFree(xid); - } else if (xmlStrEqual(name, BAD_CAST "relation")) { - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); - assert(xid); - osmdata->osm_id = strtoosmid((char *)xid, NULL, 10); - osmdata->action = ParseAction( reader, osmdata ); - - if (osmdata->osm_id > osmdata->max_rel) - osmdata->max_rel = osmdata->osm_id; - - if (osmdata->count_rel == 0) { - time(&osmdata->start_rel); - } - osmdata->count_rel++; - if (osmdata->count_rel%10 == 0) - printStatus(osmdata); - - osmdata->member_count = 0; - xmlFree(xid); - } else if (xmlStrEqual(name, BAD_CAST "member")) { - xrole = xmlTextReaderGetAttribute(reader, BAD_CAST "role"); - assert(xrole); - - xtype = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); - assert(xtype); - - xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref"); - assert(xid); - - osmdata->members[osmdata->member_count].id = strtoosmid( (char *)xid, NULL, 0 ); - osmdata->members[osmdata->member_count].role = strdup( (char *)xrole ); - - /* Currently we are only interested in 'way' members since these form polygons with holes */ - if (xmlStrEqual(xtype, BAD_CAST "way")) - osmdata->members[osmdata->member_count].type = OSMTYPE_WAY; - if (xmlStrEqual(xtype, BAD_CAST "node")) - osmdata->members[osmdata->member_count].type = OSMTYPE_NODE; - if (xmlStrEqual(xtype, BAD_CAST "relation")) - osmdata->members[osmdata->member_count].type = OSMTYPE_RELATION; - osmdata->member_count++; - - if( osmdata->member_count >= osmdata->member_max ) - realloc_members(osmdata); - xmlFree(xid); - xmlFree(xrole); - xmlFree(xtype); - } else if (xmlStrEqual(name, BAD_CAST "add") || - xmlStrEqual(name, BAD_CAST "create")) { - osmdata->action = ACTION_MODIFY; /* Turns all creates into modifies, makes it resiliant against inconsistant snapshots. */ - } else if (xmlStrEqual(name, BAD_CAST "modify")) { - osmdata->action = ACTION_MODIFY; - } else if (xmlStrEqual(name, BAD_CAST "delete")) { - osmdata->action = ACTION_DELETE; - } else if (xmlStrEqual(name, BAD_CAST "bound")) { - /* ignore */ - } else if (xmlStrEqual(name, BAD_CAST "bounds")) { - /* ignore */ - } else if (xmlStrEqual(name, BAD_CAST "changeset")) { - /* ignore */ - } else { - fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); - } - - /* Collect extra attribute information and add as tags */ - if (osmdata->extra_attributes && (xmlStrEqual(name, BAD_CAST "node") || - xmlStrEqual(name, BAD_CAST "way") || - xmlStrEqual(name, BAD_CAST "relation"))) - { - xmlChar *xtmp; - - xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "user"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_user", (char *)xtmp, 0); - xmlFree(xtmp); - } - - xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "uid"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_uid", (char *)xtmp, 0); - xmlFree(xtmp); - } - - xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "version"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_version", (char *)xtmp, 0); - xmlFree(xtmp); - } - - xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "timestamp"); - if (xtmp) { - addItem(&(osmdata->tags), "osm_timestamp", (char *)xtmp, 0); - xmlFree(xtmp); - } - } -} - - -static void EndElement(const xmlChar *name, struct osmdata_t *osmdata) -{ - if (xmlStrEqual(name, BAD_CAST "node")) { - if (node_wanted(osmdata, osmdata->node_lat, osmdata->node_lon)) { - reproject(&(osmdata->node_lat), &(osmdata->node_lon)); - if( osmdata->action == ACTION_CREATE ) - osmdata->out->node_add(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags)); - else if( osmdata->action == ACTION_MODIFY ) - osmdata->out->node_modify(osmdata->osm_id, osmdata->node_lat, osmdata->node_lon, &(osmdata->tags)); - else if( osmdata->action == ACTION_DELETE ) - osmdata->out->node_delete(osmdata->osm_id); - else - { - fprintf( stderr, "Don't know action for node %" PRIdOSMID "\n", osmdata->osm_id ); - exit_nicely(); - } - } - resetList(&(osmdata->tags)); - } else if (xmlStrEqual(name, BAD_CAST "way")) { - if( osmdata->action == ACTION_CREATE ) - osmdata->out->way_add(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) ); - else if( osmdata->action == ACTION_MODIFY ) - osmdata->out->way_modify(osmdata->osm_id, osmdata->nds, osmdata->nd_count, &(osmdata->tags) ); - else if( osmdata->action == ACTION_DELETE ) - osmdata->out->way_delete(osmdata->osm_id); - else - { - fprintf( stderr, "Don't know action for way %" PRIdOSMID "\n", osmdata->osm_id ); - exit_nicely(); - } - resetList(&(osmdata->tags)); - } else if (xmlStrEqual(name, BAD_CAST "relation")) { - if( osmdata->action == ACTION_CREATE ) - osmdata->out->relation_add(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags)); - else if( osmdata->action == ACTION_MODIFY ) - osmdata->out->relation_modify(osmdata->osm_id, osmdata->members, osmdata->member_count, &(osmdata->tags)); - else if( osmdata->action == ACTION_DELETE ) - osmdata->out->relation_delete(osmdata->osm_id); - else - { - fprintf( stderr, "Don't know action for relation %" PRIdOSMID "\n", osmdata->osm_id ); - exit_nicely(); - } - resetList(&(osmdata->tags)); - resetMembers(osmdata); - } else if (xmlStrEqual(name, BAD_CAST "tag")) { - /* ignore */ - } else if (xmlStrEqual(name, BAD_CAST "nd")) { - /* ignore */ - } else if (xmlStrEqual(name, BAD_CAST "member")) { - /* ignore */ - } else if (xmlStrEqual(name, BAD_CAST "osm")) { - printStatus(osmdata); - osmdata->filetype = FILETYPE_NONE; - } else if (xmlStrEqual(name, BAD_CAST "osmChange")) { - printStatus(osmdata); - osmdata->filetype = FILETYPE_NONE; - } else if (xmlStrEqual(name, BAD_CAST "planetdiff")) { - printStatus(osmdata); - osmdata->filetype = FILETYPE_NONE; - } else if (xmlStrEqual(name, BAD_CAST "bound")) { - /* ignore */ - } else if (xmlStrEqual(name, BAD_CAST "bounds")) { - /* ignore */ - } else if (xmlStrEqual(name, BAD_CAST "changeset")) { - /* ignore */ - resetList(&(osmdata->tags)); /* We may have accumulated some tags even if we ignored the changeset */ - } else if (xmlStrEqual(name, BAD_CAST "add")) { - osmdata->action = ACTION_NONE; - } else if (xmlStrEqual(name, BAD_CAST "create")) { - osmdata->action = ACTION_NONE; - } else if (xmlStrEqual(name, BAD_CAST "modify")) { - osmdata->action = ACTION_NONE; - } else if (xmlStrEqual(name, BAD_CAST "delete")) { - osmdata->action = ACTION_NONE; - } else { - fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); - } -} - -static void processNode(xmlTextReaderPtr reader, struct osmdata_t *osmdata) { - xmlChar *name; - name = xmlTextReaderName(reader); - if (name == NULL) - name = xmlStrdup(BAD_CAST "--"); - - switch(xmlTextReaderNodeType(reader)) { - case XML_READER_TYPE_ELEMENT: - StartElement(reader, name, osmdata); - if (xmlTextReaderIsEmptyElement(reader)) - EndElement(name, osmdata); /* No end_element for self closing tags! */ - break; - case XML_READER_TYPE_END_ELEMENT: - EndElement(name, osmdata); - break; - case XML_READER_TYPE_SIGNIFICANT_WHITESPACE: - /* Ignore */ - break; - default: - fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader)); - break; - } - - xmlFree(name); -} - -int streamFileXML2(char *filename, int sanitize, struct osmdata_t *osmdata) { - xmlTextReaderPtr reader; - int ret = 0; - - if (sanitize) - reader = sanitizerOpen(filename); - else - reader = inputUTF8(filename); - - if (reader != NULL) { - ret = xmlTextReaderRead(reader); - while (ret == 1) { - processNode(reader, osmdata); - ret = xmlTextReaderRead(reader); - } - - if (ret != 0) { - fprintf(stderr, "%s : failed to parse\n", filename); - return ret; - } - - xmlFreeTextReader(reader); - } else { - fprintf(stderr, "Unable to open %s\n", filename); - return 1; - } - return 0; -} - diff -Nru osm2pgsql-0.82.0/parse-xml2.cpp osm2pgsql-0.88.1/parse-xml2.cpp --- osm2pgsql-0.82.0/parse-xml2.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse-xml2.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,414 @@ +/* +#----------------------------------------------------------------------------- +# osm2pgsql - converts planet.osm file into PostgreSQL +# compatible output suitable to be rendered by mapnik +#----------------------------------------------------------------------------- +# Original Python implementation by Artem Pavlenko +# Re-implementation by Jon Burgess, Copyright 2006 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#include +#include +#include +#include +#include + +#include "sanitizer.hpp" +#include "input.hpp" + +#include "parse-xml2.hpp" +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "util.hpp" +#include "reprojection.hpp" + +/* Parses the action="foo" tags in JOSM change files. Obvisouly not useful from osmChange files */ +actions_t parse_xml2_t::ParseAction( xmlTextReaderPtr reader) +{ + actions_t new_action; + xmlChar *action_text; + if( filetype == FILETYPE_OSMCHANGE || filetype == FILETYPE_PLANETDIFF ) + return action; + new_action = ACTION_NONE; + action_text = xmlTextReaderGetAttribute( reader, BAD_CAST "action" ); + if( action_text == NULL ) + new_action = ACTION_CREATE; + else if( strcmp((char *)action_text, "modify") == 0 ) + new_action = ACTION_MODIFY; + else if( strcmp((char *)action_text, "delete") == 0 ) + new_action = ACTION_DELETE; + else + { + fprintf( stderr, "Unknown value for action: %s\n", (char*)action_text ); + util::exit_nicely(); + } + return new_action; +} + +void parse_xml2_t::SetFiletype(const xmlChar* name, osmdata_t* osmdata) +{ + if (xmlStrEqual(name, BAD_CAST "osm")) + { + filetype = FILETYPE_OSM; + action = ACTION_CREATE; + } + else if (xmlStrEqual(name, BAD_CAST "osmChange")) + { + filetype = FILETYPE_OSMCHANGE; + action = ACTION_NONE; + } + else if (xmlStrEqual(name, BAD_CAST "planetdiff")) + { + filetype = FILETYPE_PLANETDIFF; + action = ACTION_NONE; + } + else + { + fprintf( stderr, "Unknown XML document type: %s\n", name ); + util::exit_nicely(); + } +} + +void parse_xml2_t::StartElement(xmlTextReaderPtr reader, const xmlChar *name, osmdata_t *osmdata) +{ + xmlChar *xid, *xlat, *xlon, *xk, *xv, *xrole, *xtype; + + //first time in we figure out what kind of data this is + if (filetype == FILETYPE_NONE) + { + SetFiletype(name, osmdata); + return; + } + + //remember which this was for collecting tags at the end + bool can_have_attribs = false; + + if (xmlStrEqual(name, BAD_CAST "node")) { + can_have_attribs = true; + + xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); + xlon = xmlTextReaderGetAttribute(reader, BAD_CAST "lon"); + xlat = xmlTextReaderGetAttribute(reader, BAD_CAST "lat"); + assert(xid); + + osm_id = strtoosmid((char *)xid, NULL, 10); + action = ParseAction(reader); + + if (action != ACTION_DELETE) { + assert(xlon); assert(xlat); + node_lon = strtod((char *)xlon, NULL); + node_lat = strtod((char *)xlat, NULL); + } + + if (osm_id > max_node) + max_node = osm_id; + + if (count_node == 0) { + time(&start_node); + } + count_node++; + if (count_node%10000 == 0) + print_status(); + + xmlFree(xid); + xmlFree(xlon); + xmlFree(xlat); + } else if (xmlStrEqual(name, BAD_CAST "way")) { + can_have_attribs = true; + + xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); + assert(xid); + osm_id = strtoosmid((char *)xid, NULL, 10); + action = ParseAction( reader ); + + if (osm_id > max_way) + max_way = osm_id; + + if (count_way == 0) { + time(&start_way); + } + count_way++; + if (count_way%1000 == 0) + print_status(); + + nds.clear(); + xmlFree(xid); + + } else if (xmlStrEqual(name, BAD_CAST "relation")) { + can_have_attribs = true; + + xid = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); + assert(xid); + osm_id = strtoosmid((char *)xid, NULL, 10); + action = ParseAction( reader ); + + if (osm_id > max_rel) + max_rel = osm_id; + + if (count_rel == 0) { + time(&start_rel); + } + count_rel++; + if (count_rel%10 == 0) + print_status(); + + members.clear(); + xmlFree(xid); + } else if (xmlStrEqual(name, BAD_CAST "tag")) { + xk = xmlTextReaderGetAttribute(reader, BAD_CAST "k"); + assert(xk); + + xv = xmlTextReaderGetAttribute(reader, BAD_CAST "v"); + assert(xv); + + tags.push_back(tag((const char *)xk, (const char *)xv)); + + xmlFree(xv); + xmlFree(xk); + } else if (xmlStrEqual(name, BAD_CAST "nd")) { + xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref"); + assert(xid); + + nds.push_back(strtoosmid( (char *)xid, NULL, 10 )); + + xmlFree(xid); + } else if (xmlStrEqual(name, BAD_CAST "member")) { + xrole = xmlTextReaderGetAttribute(reader, BAD_CAST "role"); + assert(xrole); + + xtype = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); + assert(xtype); + + xid = xmlTextReaderGetAttribute(reader, BAD_CAST "ref"); + assert(xid); + + OsmType type = OSMTYPE_NODE; + if (xmlStrEqual(xtype, BAD_CAST "way")) + type = OSMTYPE_WAY; + else if (xmlStrEqual(xtype, BAD_CAST "node")) + type = OSMTYPE_NODE; + else if (xmlStrEqual(xtype, BAD_CAST "relation")) + type = OSMTYPE_RELATION; + + members.push_back(member(type, strtoosmid((char *) xid, NULL, 0), + (const char *) xrole)); + xmlFree(xid); + xmlFree(xrole); + xmlFree(xtype); + } else if (xmlStrEqual(name, BAD_CAST "add") || + xmlStrEqual(name, BAD_CAST "create")) { + action = ACTION_MODIFY; /* Turns all creates into modifies, makes it resiliant against inconsistant snapshots. */ + } else if (xmlStrEqual(name, BAD_CAST "modify")) { + action = ACTION_MODIFY; + } else if (xmlStrEqual(name, BAD_CAST "delete")) { + action = ACTION_DELETE; + } else if (xmlStrEqual(name, BAD_CAST "bound")) { + /* ignore */ + } else if (xmlStrEqual(name, BAD_CAST "bounds")) { + /* ignore */ + } else if (xmlStrEqual(name, BAD_CAST "changeset")) { + /* ignore */ + } else { + fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); + } + + /* Collect extra attribute information and add as tags */ + if (extra_attributes && can_have_attribs) + { + xmlChar *xtmp; + + xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "user"); + if (xtmp) { + tags.push_back(tag("osm_user", (const char *)xtmp)); + xmlFree(xtmp); + } + + xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "uid"); + if (xtmp) { + tags.push_back(tag("osm_uid", (const char *)xtmp)); + xmlFree(xtmp); + } + + xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "version"); + if (xtmp) { + tags.push_back(tag("osm_version", (const char *)xtmp)); + xmlFree(xtmp); + } + + xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "timestamp"); + if (xtmp) { + tags.push_back(tag("osm_timestamp", (const char *)xtmp)); + xmlFree(xtmp); + } + + xtmp = xmlTextReaderGetAttribute(reader, BAD_CAST "changeset"); + if (xtmp) { + tags.push_back(tag("osm_changeset", (const char *)xtmp)); + xmlFree(xtmp); + } + } +} + + +void parse_xml2_t::EndElement(const xmlChar *name, osmdata_t *osmdata) +{ + if (xmlStrEqual(name, BAD_CAST "node")) { + if (node_wanted(node_lat, node_lon)) { + proj->reproject(&(node_lat), &(node_lon)); + if( action == ACTION_CREATE ) + osmdata->node_add(osm_id, node_lat, node_lon, tags); + else if( action == ACTION_MODIFY ) + osmdata->node_modify(osm_id, node_lat, node_lon, tags); + else if( action == ACTION_DELETE ) + osmdata->node_delete(osm_id); + else + { + fprintf( stderr, "Don't know action for node %" PRIdOSMID "\n", osm_id ); + util::exit_nicely(); + } + } + tags.clear(); + } else if (xmlStrEqual(name, BAD_CAST "way")) { + if( action == ACTION_CREATE ) + osmdata->way_add(osm_id, nds, tags); + else if( action == ACTION_MODIFY ) + osmdata->way_modify(osm_id, nds, tags); + else if( action == ACTION_DELETE ) + osmdata->way_delete(osm_id); + else + { + fprintf( stderr, "Don't know action for way %" PRIdOSMID "\n", osm_id ); + util::exit_nicely(); + } + tags.clear(); + } else if (xmlStrEqual(name, BAD_CAST "relation")) { + if( action == ACTION_CREATE ) + osmdata->relation_add(osm_id, members, tags); + else if( action == ACTION_MODIFY ) + osmdata->relation_modify(osm_id, members, tags); + else if( action == ACTION_DELETE ) + osmdata->relation_delete(osm_id); + else + { + fprintf( stderr, "Don't know action for relation %" PRIdOSMID "\n", osm_id ); + util::exit_nicely(); + } + tags.clear(); + members.clear(); + } else if (xmlStrEqual(name, BAD_CAST "tag")) { + /* ignore */ + } else if (xmlStrEqual(name, BAD_CAST "nd")) { + /* ignore */ + } else if (xmlStrEqual(name, BAD_CAST "member")) { + /* ignore */ + } else if (xmlStrEqual(name, BAD_CAST "osm")) { + print_status(); + filetype = FILETYPE_NONE; + } else if (xmlStrEqual(name, BAD_CAST "osmChange")) { + print_status(); + filetype = FILETYPE_NONE; + } else if (xmlStrEqual(name, BAD_CAST "planetdiff")) { + print_status(); + filetype = FILETYPE_NONE; + } else if (xmlStrEqual(name, BAD_CAST "bound")) { + /* ignore */ + } else if (xmlStrEqual(name, BAD_CAST "bounds")) { + /* ignore */ + } else if (xmlStrEqual(name, BAD_CAST "changeset")) { + /* ignore */ + tags.clear(); /* We may have accumulated some tags even if we ignored the changeset */ + } else if (xmlStrEqual(name, BAD_CAST "add")) { + action = ACTION_NONE; + } else if (xmlStrEqual(name, BAD_CAST "create")) { + action = ACTION_NONE; + } else if (xmlStrEqual(name, BAD_CAST "modify")) { + action = ACTION_NONE; + } else if (xmlStrEqual(name, BAD_CAST "delete")) { + action = ACTION_NONE; + } else { + fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); + } +} + +void parse_xml2_t::processNode(xmlTextReaderPtr reader, osmdata_t *osmdata) { + xmlChar *name; + name = xmlTextReaderName(reader); + if (name == NULL) + name = xmlStrdup(BAD_CAST "--"); + + switch(xmlTextReaderNodeType(reader)) { + case XML_READER_TYPE_ELEMENT: + StartElement(reader, name, osmdata); + if (xmlTextReaderIsEmptyElement(reader)) + EndElement(name, osmdata); /* No end_element for self closing tags! */ + break; + case XML_READER_TYPE_END_ELEMENT: + EndElement(name, osmdata); + break; + case XML_READER_TYPE_SIGNIFICANT_WHITESPACE: + /* Ignore */ + break; + default: + fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader)); + break; + } + + xmlFree(name); +} + +parse_xml2_t::parse_xml2_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon, const double minlat, const double maxlon, const double maxlat): + parse_t(extra_attributes_, bbox_, projection_, minlon, minlat, maxlon, maxlat) +{ + LIBXML_TEST_VERSION; +} + +parse_xml2_t::~parse_xml2_t() +{ + xmlCleanupParser(); + xmlMemoryDump(); +} + +int parse_xml2_t::streamFile(const char *filename, const int sanitize, osmdata_t *osmdata) { + xmlTextReaderPtr reader; + int ret = 0; + + if (sanitize) + reader = sanitizerOpen(filename); + else + reader = inputUTF8(filename); + + if (reader != NULL) { + ret = xmlTextReaderRead(reader); + while (ret == 1) { + processNode(reader, osmdata); + ret = xmlTextReaderRead(reader); + } + + if (ret != 0) { + fprintf(stderr, "%s : failed to parse\n", filename); + return ret; + } + + xmlFreeTextReader(reader); + } else { + fprintf(stderr, "Unable to open %s\n", filename); + return 1; + } + return 0; +} diff -Nru osm2pgsql-0.82.0/parse-xml2.h osm2pgsql-0.88.1/parse-xml2.h --- osm2pgsql-0.82.0/parse-xml2.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/parse-xml2.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,30 +0,0 @@ -/* -#----------------------------------------------------------------------------- -# osm2pgsql - converts planet.osm file into PostgreSQL -# compatible output suitable to be rendered by mapnik -#----------------------------------------------------------------------------- -# Original Python implementation by Artem Pavlenko -# Re-implementation by Jon Burgess, Copyright 2006 -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -*/ - -#ifndef PARSE_XML2_H -#define PARSE_XML2_h - -int streamFileXML2(char *filename, int sanitize, struct osmdata_t *osmdata); - -#endif diff -Nru osm2pgsql-0.82.0/parse-xml2.hpp osm2pgsql-0.88.1/parse-xml2.hpp --- osm2pgsql-0.82.0/parse-xml2.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/parse-xml2.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,62 @@ +/* +#----------------------------------------------------------------------------- +# osm2pgsql - converts planet.osm file into PostgreSQL +# compatible output suitable to be rendered by mapnik +#----------------------------------------------------------------------------- +# Original Python implementation by Artem Pavlenko +# Re-implementation by Jon Burgess, Copyright 2006 +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +#----------------------------------------------------------------------------- +*/ + +#ifndef PARSE_XML2_H +#define PARSE_XML2_H + +/* Open and incrementally read an XML file. + * + * Parameters: + * - filename: the path to the XML file to stream. + * - sanitize: if non-zero, use a parser which attempts to sanitize bad UTF-8 + * characters. + * - osmdata: as the file is parsed, it will call functions on `osmdata->out` + * based on the action and type of object, e.g: `node_add`, `relation_modify`, + * and so on. + * + * Return value: 0 on success. A non-zero value indicates failure. + */ + +#include "parse.hpp" + +#include + +class parse_xml2_t: public parse_t +{ +public: + parse_xml2_t(const int extra_attributes_, const bool bbox_, const boost::shared_ptr& projection_, + const double minlon, const double minlat, const double maxlon, const double maxlat); + virtual ~parse_xml2_t(); + virtual int streamFile(const char *filename, const int sanitize, osmdata_t *osmdata); +protected: + parse_xml2_t(); + actions_t ParseAction( xmlTextReaderPtr reader); + void SetFiletype(const xmlChar* name, osmdata_t* osmdata); + void StartElement(xmlTextReaderPtr reader, const xmlChar *name, osmdata_t *osmdata); + void EndElement(const xmlChar *name, osmdata_t *osmdata); + void processNode(xmlTextReaderPtr reader, osmdata_t *osmdata); +}; + + +#endif diff -Nru osm2pgsql-0.82.0/pgsql.c osm2pgsql-0.88.1/pgsql.c --- osm2pgsql-0.82.0/pgsql.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/pgsql.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,133 +0,0 @@ -/* Helper functions for the postgresql connections */ - -#include -#include -#include -#include -#include -#include "osmtypes.h" /* For exit_nicely() */ -#include "pgsql.h" - -void escape(char *out, int len, const char *in) -{ - /* Apply escaping of TEXT COPY data - Escape: backslash itself, newline, carriage return, and the current delimiter character (tab) - file:///usr/share/doc/postgresql-8.1.8/html/sql-copy.html - */ - int count = 0; - const char *old_in = in, *old_out = out; - - if (!len) - return; - - while(*in && count < len-3) { - switch(*in) { - case '\\': *out++ = '\\'; *out++ = '\\'; count+= 2; break; - /* case 8: *out++ = '\\'; *out++ = '\b'; count+= 2; break; */ - /* case 12: *out++ = '\\'; *out++ = '\f'; count+= 2; break; */ - case '\n': *out++ = '\\'; *out++ = '\n'; count+= 2; break; - case '\r': *out++ = '\\'; *out++ = '\r'; count+= 2; break; - case '\t': *out++ = '\\'; *out++ = '\t'; count+= 2; break; - /* case 11: *out++ = '\\'; *out++ = '\v'; count+= 2; break; */ - default: *out++ = *in; count++; break; - } - in++; - } - *out = '\0'; - - if (*in) - fprintf(stderr, "%s truncated at %d chars: %s\n%s\n", __FUNCTION__, count, old_in, old_out); -} - -int pgsql_exec(PGconn *sql_conn, ExecStatusType expect, const char *fmt, ...) -{ - PGresult *res; - va_list ap; - char *sql, *nsql; - int n, size = 100; - - /* Based on vprintf manual page */ - /* Guess we need no more than 100 bytes. */ - - if ((sql = malloc(size)) == NULL) { - fprintf(stderr, "Memory allocation failed\n"); - exit_nicely(); - } - - while (1) { - /* Try to print in the allocated space. */ - va_start(ap, fmt); - n = vsnprintf(sql, size, fmt, ap); - va_end(ap); - /* If that worked, return the string. */ - if (n > -1 && n < size) - break; - /* Else try again with more space. */ - if (n > -1) /* glibc 2.1 */ - size = n+1; /* precisely what is needed */ - else /* glibc 2.0 */ - size *= 2; /* twice the old size */ - if ((nsql = realloc (sql, size)) == NULL) { - free(sql); - fprintf(stderr, "Memory re-allocation failed\n"); - exit_nicely(); - } else { - sql = nsql; - } - } - -#ifdef DEBUG_PGSQL - fprintf( stderr, "Executing: %s\n", sql ); -#endif - res = PQexec(sql_conn, sql); - if (PQresultStatus(res) != expect) { - fprintf(stderr, "%s failed: %s\n", sql, PQerrorMessage(sql_conn)); - free(sql); - PQclear(res); - exit_nicely(); - } - free(sql); - PQclear(res); - return 0; -} - -int pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql) -{ -#ifdef DEBUG_PGSQL - fprintf( stderr, "%s>>> %s\n", context, sql ); -#endif - int r = PQputCopyData(sql_conn, sql, strlen(sql)); - if (r != 1) { - fprintf(stderr, "%s - bad result during COPY, data %s\n", context, sql); - exit_nicely(); - } - return 0; -} - -PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, int nParams, const char *const * paramValues, ExecStatusType expect) -{ -#ifdef DEBUG_PGSQL - fprintf( stderr, "ExecPrepared: %s\n", stmtName ); -#endif - PGresult *res = PQexecPrepared(sql_conn, stmtName, nParams, paramValues, NULL, NULL, 0); - if (PQresultStatus(res) != expect) { - fprintf(stderr, "%s failed: %s(%d)\n", stmtName, PQerrorMessage(sql_conn), PQresultStatus(res)); - if( nParams ) - { - int i; - fprintf( stderr, "Arguments were: " ); - for( i=0; i +#include +#include +#include +#include + +void escape(const std::string &src, std::string &dst) +{ + BOOST_FOREACH(const char c, src) + { + switch(c) { + case '\\': dst.append("\\\\"); break; + //case 8: dst.append("\\\b"); break; + //case 12: dst.append("\\\f"); break; + case '\n': dst.append("\\\n"); break; + case '\r': dst.append("\\\r"); break; + case '\t': dst.append("\\\t"); break; + //case 11: dst.append("\\\v"); break; + default: dst.push_back(c); break; + } + } +} + + +boost::shared_ptr pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql) +{ + return pgsql_exec_simple(sql_conn, expect, sql.c_str()); +} + +boost::shared_ptr pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const char *sql) +{ + PGresult* res; +#ifdef DEBUG_PGSQL + fprintf( stderr, "Executing: %s\n", sql ); +#endif + res = PQexec(sql_conn, sql); + if (PQresultStatus(res) != expect) { + PQclear(res); + throw std::runtime_error((boost::format("%1% failed: %2%\n") % sql % PQerrorMessage(sql_conn)).str()); + } + return boost::shared_ptr(res, &PQclear); +} + +int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, ...) +{ + + va_list ap; + char *sql, *nsql; + int n, size = 100; + + /* Based on vprintf manual page */ + /* Guess we need no more than 100 bytes. */ + + if ((sql = (char *)malloc(size)) == NULL) + throw std::runtime_error("Memory allocation failed in pgsql_exec"); + + while (1) { + /* Try to print in the allocated space. */ + va_start(ap, fmt); + n = vsnprintf(sql, size, fmt, ap); + va_end(ap); + /* If that worked, return the string. */ + if (n > -1 && n < size) + break; + /* Else try again with more space. */ + if (n > -1) /* glibc 2.1 */ + size = n+1; /* precisely what is needed */ + else /* glibc 2.0 */ + size *= 2; /* twice the old size */ + if ((nsql = (char *)realloc (sql, size)) == NULL) { + free(sql); + throw std::runtime_error("Memory re-allocation failed in pgsql_exec"); + } else { + sql = nsql; + } + } + +#ifdef DEBUG_PGSQL + fprintf( stderr, "Executing: %s\n", sql ); +#endif + PGresult* res = PQexec(sql_conn, sql); + if (PQresultStatus(res) != expect) { + std::string err_msg = (boost::format("%1% failed: %2%") % sql % PQerrorMessage(sql_conn)).str(); + free(sql); + PQclear(res); + throw std::runtime_error(err_msg); + } + free(sql); + PQclear(res); + return 0; +} + +void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql, int len) +{ +#ifdef DEBUG_PGSQL + fprintf(stderr, "%s>>> %s\n", context, sql ); +#endif + int r = PQputCopyData(sql_conn, sql, len); + switch(r) + { + //need to wait for write ready + case 0: + throw std::runtime_error((boost::format("%1% - bad result during COPY, data %2%") % context % sql % PQerrorMessage(sql_conn)).str()); + break; + //error occurred + case -1: + throw std::runtime_error((boost::format("%1%: %2% - bad result during COPY, data %3%") % PQerrorMessage(sql_conn) % context % sql).str()); + break; + //other possibility is 1 which means success + } +} + +PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int nParams, const char *const * paramValues, const ExecStatusType expect) +{ +#ifdef DEBUG_PGSQL + fprintf( stderr, "ExecPrepared: %s\n", stmtName ); +#endif + //run the prepared statement + PGresult *res = PQexecPrepared(sql_conn, stmtName, nParams, paramValues, NULL, NULL, 0); + if(PQresultStatus(res) != expect) + { + std::string message = (boost::format("%1% failed: %2%(%3%)\n") % stmtName % PQerrorMessage(sql_conn) % PQresultStatus(res)).str(); + if(nParams) + { + message += "Arguments were: "; + for(int i = 0; i < nParams; i++) + { + message += paramValues[i]; + message += ", "; + } + } + PQclear(res); + throw std::runtime_error(message); + } + + //TODO: this seems a bit strange + //if you decided you wanted to expect something other than this you didnt want to use the result? + if( expect != PGRES_TUPLES_OK ) + { + PQclear(res); + res = NULL; + } + return res; +} diff -Nru osm2pgsql-0.82.0/pgsql.h osm2pgsql-0.88.1/pgsql.h --- osm2pgsql-0.82.0/pgsql.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/pgsql.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ -/* Helper functions for pgsql access */ - -/* Current middle and output-pgsql do a lot of things similarly, this should - * be used to abstract to commonalities */ - -PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, int nParams, const char *const * paramValues, ExecStatusType expect); -int pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql); -int pgsql_exec(PGconn *sql_conn, ExecStatusType expect, const char *fmt, ...) __attribute__ ((format (printf, 3, 4))); -void escape(char *out, int len, const char *in); diff -Nru osm2pgsql-0.82.0/pgsql.hpp osm2pgsql-0.88.1/pgsql.hpp --- osm2pgsql-0.82.0/pgsql.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/pgsql.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,34 @@ +/* Helper functions for pgsql access */ + +/* Current middle and output-pgsql do a lot of things similarly, this should + * be used to abstract to commonalities */ + +#ifndef PGSQL_H +#define PGSQL_H + +#include +#include +#include +#include + +PGresult *pgsql_execPrepared( PGconn *sql_conn, const char *stmtName, const int nParams, const char *const * paramValues, const ExecStatusType expect); +void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql, int len); +boost::shared_ptr pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const std::string& sql); +boost::shared_ptr pgsql_exec_simple(PGconn *sql_conn, const ExecStatusType expect, const char *sql); +int pgsql_exec(PGconn *sql_conn, const ExecStatusType expect, const char *fmt, ...) +#ifndef _MSC_VER + __attribute__ ((format (printf, 3, 4))) +#endif +; + +void escape(const std::string &src, std::string& dst); + + +inline void pgsql_CopyData(const char *context, PGconn *sql_conn, const char *sql) { + pgsql_CopyData(context, sql_conn, sql, strlen(sql)); +} + +inline void pgsql_CopyData(const char *context, PGconn *sql_conn, const std::string &sql) { + pgsql_CopyData(context, sql_conn, sql.c_str(), sql.length()); +} +#endif diff -Nru osm2pgsql-0.82.0/processor-line.cpp osm2pgsql-0.88.1/processor-line.cpp --- osm2pgsql-0.82.0/processor-line.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/processor-line.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,17 @@ +#include "processor-line.hpp" + +processor_line::processor_line(int srid) : geometry_processor(srid, "LINESTRING", interest_way) +{ +} + +processor_line::~processor_line() +{ +} + +geometry_builder::maybe_wkt_t processor_line::process_way(const nodelist_t &nodes) +{ + //have the builder make the wkt + geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodes, false); + //hand back the wkt + return wkt; +} diff -Nru osm2pgsql-0.82.0/processor-line.hpp osm2pgsql-0.88.1/processor-line.hpp --- osm2pgsql-0.82.0/processor-line.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/processor-line.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,16 @@ +#ifndef PROCESSOR_LINE_HPP +#define PROCESSOR_LINE_HPP + +#include "geometry-processor.hpp" + +struct processor_line : public geometry_processor { + processor_line(int srid); + virtual ~processor_line(); + + geometry_builder::maybe_wkt_t process_way(const nodelist_t &nodes); + +private: + geometry_builder builder; +}; + +#endif /* PROCESSOR_LINE_HPP */ diff -Nru osm2pgsql-0.82.0/processor-point.cpp osm2pgsql-0.88.1/processor-point.cpp --- osm2pgsql-0.82.0/processor-point.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/processor-point.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,18 @@ +#include "config.h" // for FIXED_POINT +#include "processor-point.hpp" +#include "util.hpp" + +#include + +processor_point::processor_point(int srid) + : geometry_processor(srid, "POINT", interest_node) { +} + +processor_point::~processor_point() { +} + +geometry_builder::maybe_wkt_t processor_point::process_node(double lat, double lon) { + geometry_builder::maybe_wkt_t wkt(new geometry_builder::wkt_t()); + wkt->geom = (boost::format("POINT(%.15g %.15g)") % lon % lat).str(); + return wkt; +} diff -Nru osm2pgsql-0.82.0/processor-point.hpp osm2pgsql-0.88.1/processor-point.hpp --- osm2pgsql-0.82.0/processor-point.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/processor-point.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,13 @@ +#ifndef PROCESSOR_POINT_HPP +#define PROCESSOR_POINT_HPP + +#include "geometry-processor.hpp" + +struct processor_point : public geometry_processor { + processor_point(int srid); + virtual ~processor_point(); + + geometry_builder::maybe_wkt_t process_node(double lat, double lon); +}; + +#endif /* PROCESSOR_POINT_HPP */ diff -Nru osm2pgsql-0.82.0/processor-polygon.cpp osm2pgsql-0.88.1/processor-polygon.cpp --- osm2pgsql-0.82.0/processor-polygon.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/processor-polygon.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,24 @@ +#include "processor-polygon.hpp" + +processor_polygon::processor_polygon(int srid, bool enable_multi) : geometry_processor(srid, "GEOMETRY", interest_way | interest_relation), enable_multi(enable_multi) +{ +} + +processor_polygon::~processor_polygon() +{ +} + +geometry_builder::maybe_wkt_t processor_polygon::process_way(const nodelist_t &nodes) +{ + //have the builder make the wkt + geometry_builder::maybe_wkt_t wkt = builder.get_wkt_simple(nodes, true); + //hand back the wkt + return wkt; +} + +geometry_builder::maybe_wkts_t processor_polygon::process_relation(const multinodelist_t &nodes) +{ + //the hard word was already done for us in getting at the node data for each way. at this point just make the geom + geometry_builder::maybe_wkts_t wkts = builder.build_polygons(nodes, enable_multi, -1); + return wkts; +} diff -Nru osm2pgsql-0.82.0/processor-polygon.hpp osm2pgsql-0.88.1/processor-polygon.hpp --- osm2pgsql-0.82.0/processor-polygon.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/processor-polygon.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,18 @@ +#ifndef PROCESSOR_POLYGON_HPP +#define PROCESSOR_POLYGON_HPP + +#include "geometry-processor.hpp" + +struct processor_polygon : public geometry_processor { + processor_polygon(int srid, bool enable_multi); + virtual ~processor_polygon(); + + geometry_builder::maybe_wkt_t process_way(const nodelist_t &nodes); + geometry_builder::maybe_wkts_t process_relation(const multinodelist_t &nodes); + +private: + bool enable_multi; + geometry_builder builder; +}; + +#endif /* PROCESSOR_POLYGON_HPP */ diff -Nru osm2pgsql-0.82.0/rb.c osm2pgsql-0.88.1/rb.c --- osm2pgsql-0.82.0/rb.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/rb.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,929 +0,0 @@ -/* Produced by texiweb from libavl.w. */ - -/* libavl - library for manipulation of binary trees. - Copyright (C) 1998-2002, 2004 Free Software Foundation, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. - - The author may be contacted at on the Internet, or - write to Ben Pfaff, Stanford University, Computer Science Dept., 353 - Serra Mall, Stanford CA 94305, USA. -*/ - -#include -#include -#include -#include -#include "rb.h" - -/* Creates and returns a new table - with comparison function |compare| using parameter |param| - and memory allocator |allocator|. - Returns |NULL| if memory allocation failed. */ -struct rb_table * -rb_create (rb_comparison_func *compare, void *param, - struct libavl_allocator *allocator) -{ - struct rb_table *tree; - - assert (compare != NULL); - - if (allocator == NULL) - allocator = &rb_allocator_default; - - tree = allocator->libavl_malloc (allocator, sizeof *tree); - if (tree == NULL) - return NULL; - - tree->rb_root = NULL; - tree->rb_compare = compare; - tree->rb_param = param; - tree->rb_alloc = allocator; - tree->rb_count = 0; - tree->rb_generation = 0; - - return tree; -} - -/* Search |tree| for an item matching |item|, and return it if found. - Otherwise return |NULL|. */ -void * -rb_find (const struct rb_table *tree, const void *item) -{ - const struct rb_node *p; - - assert (tree != NULL && item != NULL); - for (p = tree->rb_root; p != NULL; ) - { - int cmp = tree->rb_compare (item, p->rb_data, tree->rb_param); - - if (cmp < 0) - p = p->rb_link[0]; - else if (cmp > 0) - p = p->rb_link[1]; - else /* |cmp == 0| */ - return p->rb_data; - } - - return NULL; -} - -/* Inserts |item| into |tree| and returns a pointer to |item|'s address. - If a duplicate item is found in the tree, - returns a pointer to the duplicate without inserting |item|. - Returns |NULL| in case of memory allocation failure. */ -void ** -rb_probe (struct rb_table *tree, void *item) -{ - struct rb_node *pa[RB_MAX_HEIGHT]; /* Nodes on stack. */ - unsigned char da[RB_MAX_HEIGHT]; /* Directions moved from stack nodes. */ - int k; /* Stack height. */ - - struct rb_node *p; /* Traverses tree looking for insertion point. */ - struct rb_node *n; /* Newly inserted node. */ - - assert (tree != NULL && item != NULL); - - pa[0] = (struct rb_node *) &tree->rb_root; - da[0] = 0; - k = 1; - for (p = tree->rb_root; p != NULL; p = p->rb_link[da[k - 1]]) - { - int cmp = tree->rb_compare (item, p->rb_data, tree->rb_param); - if (cmp == 0) - return &p->rb_data; - - pa[k] = p; - da[k++] = cmp > 0; - } - - n = pa[k - 1]->rb_link[da[k - 1]] = - tree->rb_alloc->libavl_malloc (tree->rb_alloc, sizeof *n); - if (n == NULL) - return NULL; - - n->rb_data = item; - n->rb_link[0] = n->rb_link[1] = NULL; - n->rb_color = RB_RED; - tree->rb_count++; - tree->rb_generation++; - - while (k >= 3 && pa[k - 1]->rb_color == RB_RED) - { - if (da[k - 2] == 0) - { - struct rb_node *y = pa[k - 2]->rb_link[1]; - if (y != NULL && y->rb_color == RB_RED) - { - pa[k - 1]->rb_color = y->rb_color = RB_BLACK; - pa[k - 2]->rb_color = RB_RED; - k -= 2; - } - else - { - struct rb_node *x; - - if (da[k - 1] == 0) - y = pa[k - 1]; - else - { - x = pa[k - 1]; - y = x->rb_link[1]; - x->rb_link[1] = y->rb_link[0]; - y->rb_link[0] = x; - pa[k - 2]->rb_link[0] = y; - } - - x = pa[k - 2]; - x->rb_color = RB_RED; - y->rb_color = RB_BLACK; - - x->rb_link[0] = y->rb_link[1]; - y->rb_link[1] = x; - pa[k - 3]->rb_link[da[k - 3]] = y; - break; - } - } - else - { - struct rb_node *y = pa[k - 2]->rb_link[0]; - if (y != NULL && y->rb_color == RB_RED) - { - pa[k - 1]->rb_color = y->rb_color = RB_BLACK; - pa[k - 2]->rb_color = RB_RED; - k -= 2; - } - else - { - struct rb_node *x; - - if (da[k - 1] == 1) - y = pa[k - 1]; - else - { - x = pa[k - 1]; - y = x->rb_link[0]; - x->rb_link[0] = y->rb_link[1]; - y->rb_link[1] = x; - pa[k - 2]->rb_link[1] = y; - } - - x = pa[k - 2]; - x->rb_color = RB_RED; - y->rb_color = RB_BLACK; - - x->rb_link[1] = y->rb_link[0]; - y->rb_link[0] = x; - pa[k - 3]->rb_link[da[k - 3]] = y; - break; - } - } - } - tree->rb_root->rb_color = RB_BLACK; - - - return &n->rb_data; -} - -/* Inserts |item| into |table|. - Returns |NULL| if |item| was successfully inserted - or if a memory allocation error occurred. - Otherwise, returns the duplicate item. */ -void * -rb_insert (struct rb_table *table, void *item) -{ - void **p = rb_probe (table, item); - return p == NULL || *p == item ? NULL : *p; -} - -/* Inserts |item| into |table|, replacing any duplicate item. - Returns |NULL| if |item| was inserted without replacing a duplicate, - or if a memory allocation error occurred. - Otherwise, returns the item that was replaced. */ -void * -rb_replace (struct rb_table *table, void *item) -{ - void **p = rb_probe (table, item); - if (p == NULL || *p == item) - return NULL; - else - { - void *r = *p; - *p = item; - return r; - } -} - -/* Deletes from |tree| and returns an item matching |item|. - Returns a null pointer if no matching item found. */ -void * -rb_delete (struct rb_table *tree, const void *item) -{ - struct rb_node *pa[RB_MAX_HEIGHT]; /* Nodes on stack. */ - unsigned char da[RB_MAX_HEIGHT]; /* Directions moved from stack nodes. */ - int k; /* Stack height. */ - - struct rb_node *p; /* The node to delete, or a node part way to it. */ - int cmp; /* Result of comparison between |item| and |p|. */ - - assert (tree != NULL && item != NULL); - - k = 0; - p = (struct rb_node *) &tree->rb_root; - for (cmp = -1; cmp != 0; - cmp = tree->rb_compare (item, p->rb_data, tree->rb_param)) - { - int dir = cmp > 0; - - pa[k] = p; - da[k++] = dir; - - p = p->rb_link[dir]; - if (p == NULL) - return NULL; - } - item = p->rb_data; - - if (p->rb_link[1] == NULL) - pa[k - 1]->rb_link[da[k - 1]] = p->rb_link[0]; - else - { - enum rb_color t; - struct rb_node *r = p->rb_link[1]; - - if (r->rb_link[0] == NULL) - { - r->rb_link[0] = p->rb_link[0]; - t = r->rb_color; - r->rb_color = p->rb_color; - p->rb_color = t; - pa[k - 1]->rb_link[da[k - 1]] = r; - da[k] = 1; - pa[k++] = r; - } - else - { - struct rb_node *s; - int j = k++; - - for (;;) - { - da[k] = 0; - pa[k++] = r; - s = r->rb_link[0]; - if (s->rb_link[0] == NULL) - break; - - r = s; - } - - da[j] = 1; - pa[j] = s; - pa[j - 1]->rb_link[da[j - 1]] = s; - - s->rb_link[0] = p->rb_link[0]; - r->rb_link[0] = s->rb_link[1]; - s->rb_link[1] = p->rb_link[1]; - - t = s->rb_color; - s->rb_color = p->rb_color; - p->rb_color = t; - } - } - - if (p->rb_color == RB_BLACK) - { - for (;;) - { - struct rb_node *x = pa[k - 1]->rb_link[da[k - 1]]; - if (x != NULL && x->rb_color == RB_RED) - { - x->rb_color = RB_BLACK; - break; - } - if (k < 2) - break; - - if (da[k - 1] == 0) - { - struct rb_node *w = pa[k - 1]->rb_link[1]; - - if (w->rb_color == RB_RED) - { - w->rb_color = RB_BLACK; - pa[k - 1]->rb_color = RB_RED; - - pa[k - 1]->rb_link[1] = w->rb_link[0]; - w->rb_link[0] = pa[k - 1]; - pa[k - 2]->rb_link[da[k - 2]] = w; - - pa[k] = pa[k - 1]; - da[k] = 0; - pa[k - 1] = w; - k++; - - w = pa[k - 1]->rb_link[1]; - } - - if ((w->rb_link[0] == NULL - || w->rb_link[0]->rb_color == RB_BLACK) - && (w->rb_link[1] == NULL - || w->rb_link[1]->rb_color == RB_BLACK)) - w->rb_color = RB_RED; - else - { - if (w->rb_link[1] == NULL - || w->rb_link[1]->rb_color == RB_BLACK) - { - struct rb_node *y = w->rb_link[0]; - y->rb_color = RB_BLACK; - w->rb_color = RB_RED; - w->rb_link[0] = y->rb_link[1]; - y->rb_link[1] = w; - w = pa[k - 1]->rb_link[1] = y; - } - - w->rb_color = pa[k - 1]->rb_color; - pa[k - 1]->rb_color = RB_BLACK; - w->rb_link[1]->rb_color = RB_BLACK; - - pa[k - 1]->rb_link[1] = w->rb_link[0]; - w->rb_link[0] = pa[k - 1]; - pa[k - 2]->rb_link[da[k - 2]] = w; - break; - } - } - else - { - struct rb_node *w = pa[k - 1]->rb_link[0]; - - if (w->rb_color == RB_RED) - { - w->rb_color = RB_BLACK; - pa[k - 1]->rb_color = RB_RED; - - pa[k - 1]->rb_link[0] = w->rb_link[1]; - w->rb_link[1] = pa[k - 1]; - pa[k - 2]->rb_link[da[k - 2]] = w; - - pa[k] = pa[k - 1]; - da[k] = 1; - pa[k - 1] = w; - k++; - - w = pa[k - 1]->rb_link[0]; - } - - if ((w->rb_link[0] == NULL - || w->rb_link[0]->rb_color == RB_BLACK) - && (w->rb_link[1] == NULL - || w->rb_link[1]->rb_color == RB_BLACK)) - w->rb_color = RB_RED; - else - { - if (w->rb_link[0] == NULL - || w->rb_link[0]->rb_color == RB_BLACK) - { - struct rb_node *y = w->rb_link[1]; - y->rb_color = RB_BLACK; - w->rb_color = RB_RED; - w->rb_link[1] = y->rb_link[0]; - y->rb_link[0] = w; - w = pa[k - 1]->rb_link[0] = y; - } - - w->rb_color = pa[k - 1]->rb_color; - pa[k - 1]->rb_color = RB_BLACK; - w->rb_link[0]->rb_color = RB_BLACK; - - pa[k - 1]->rb_link[0] = w->rb_link[1]; - w->rb_link[1] = pa[k - 1]; - pa[k - 2]->rb_link[da[k - 2]] = w; - break; - } - } - - k--; - } - - } - - tree->rb_alloc->libavl_free (tree->rb_alloc, p); - tree->rb_count--; - tree->rb_generation++; - return (void *) item; -} - -/* Refreshes the stack of parent pointers in |trav| - and updates its generation number. */ -static void -trav_refresh (struct rb_traverser *trav) -{ - assert (trav != NULL); - - trav->rb_generation = trav->rb_table->rb_generation; - - if (trav->rb_node != NULL) - { - rb_comparison_func *cmp = trav->rb_table->rb_compare; - void *param = trav->rb_table->rb_param; - struct rb_node *node = trav->rb_node; - struct rb_node *i; - - trav->rb_height = 0; - for (i = trav->rb_table->rb_root; i != node; ) - { - assert (trav->rb_height < RB_MAX_HEIGHT); - assert (i != NULL); - - trav->rb_stack[trav->rb_height++] = i; - i = i->rb_link[cmp (node->rb_data, i->rb_data, param) > 0]; - } - } -} - -/* Initializes |trav| for use with |tree| - and selects the null node. */ -void -rb_t_init (struct rb_traverser *trav, struct rb_table *tree) -{ - trav->rb_table = tree; - trav->rb_node = NULL; - trav->rb_height = 0; - trav->rb_generation = tree->rb_generation; -} - -/* Initializes |trav| for |tree| - and selects and returns a pointer to its least-valued item. - Returns |NULL| if |tree| contains no nodes. */ -void * -rb_t_first (struct rb_traverser *trav, struct rb_table *tree) -{ - struct rb_node *x; - - assert (tree != NULL && trav != NULL); - - trav->rb_table = tree; - trav->rb_height = 0; - trav->rb_generation = tree->rb_generation; - - x = tree->rb_root; - if (x != NULL) - while (x->rb_link[0] != NULL) - { - assert (trav->rb_height < RB_MAX_HEIGHT); - trav->rb_stack[trav->rb_height++] = x; - x = x->rb_link[0]; - } - trav->rb_node = x; - - return x != NULL ? x->rb_data : NULL; -} - -/* Initializes |trav| for |tree| - and selects and returns a pointer to its greatest-valued item. - Returns |NULL| if |tree| contains no nodes. */ -void * -rb_t_last (struct rb_traverser *trav, struct rb_table *tree) -{ - struct rb_node *x; - - assert (tree != NULL && trav != NULL); - - trav->rb_table = tree; - trav->rb_height = 0; - trav->rb_generation = tree->rb_generation; - - x = tree->rb_root; - if (x != NULL) - while (x->rb_link[1] != NULL) - { - assert (trav->rb_height < RB_MAX_HEIGHT); - trav->rb_stack[trav->rb_height++] = x; - x = x->rb_link[1]; - } - trav->rb_node = x; - - return x != NULL ? x->rb_data : NULL; -} - -/* Searches for |item| in |tree|. - If found, initializes |trav| to the item found and returns the item - as well. - If there is no matching item, initializes |trav| to the null item - and returns |NULL|. */ -void * -rb_t_find (struct rb_traverser *trav, struct rb_table *tree, void *item) -{ - struct rb_node *p, *q; - - assert (trav != NULL && tree != NULL && item != NULL); - trav->rb_table = tree; - trav->rb_height = 0; - trav->rb_generation = tree->rb_generation; - for (p = tree->rb_root; p != NULL; p = q) - { - int cmp = tree->rb_compare (item, p->rb_data, tree->rb_param); - - if (cmp < 0) - q = p->rb_link[0]; - else if (cmp > 0) - q = p->rb_link[1]; - else /* |cmp == 0| */ - { - trav->rb_node = p; - return p->rb_data; - } - - assert (trav->rb_height < RB_MAX_HEIGHT); - trav->rb_stack[trav->rb_height++] = p; - } - - trav->rb_height = 0; - trav->rb_node = NULL; - return NULL; -} - -/* Attempts to insert |item| into |tree|. - If |item| is inserted successfully, it is returned and |trav| is - initialized to its location. - If a duplicate is found, it is returned and |trav| is initialized to - its location. No replacement of the item occurs. - If a memory allocation failure occurs, |NULL| is returned and |trav| - is initialized to the null item. */ -void * -rb_t_insert (struct rb_traverser *trav, struct rb_table *tree, void *item) -{ - void **p; - - assert (trav != NULL && tree != NULL && item != NULL); - - p = rb_probe (tree, item); - if (p != NULL) - { - trav->rb_table = tree; - trav->rb_node = - ((struct rb_node *) - ((char *) p - offsetof (struct rb_node, rb_data))); - trav->rb_generation = tree->rb_generation - 1; - return *p; - } - else - { - rb_t_init (trav, tree); - return NULL; - } -} - -/* Initializes |trav| to have the same current node as |src|. */ -void * -rb_t_copy (struct rb_traverser *trav, const struct rb_traverser *src) -{ - assert (trav != NULL && src != NULL); - - if (trav != src) - { - trav->rb_table = src->rb_table; - trav->rb_node = src->rb_node; - trav->rb_generation = src->rb_generation; - if (trav->rb_generation == trav->rb_table->rb_generation) - { - trav->rb_height = src->rb_height; - memcpy (trav->rb_stack, (const void *) src->rb_stack, - sizeof *trav->rb_stack * trav->rb_height); - } - } - - return trav->rb_node != NULL ? trav->rb_node->rb_data : NULL; -} - -/* Returns the next data item in inorder - within the tree being traversed with |trav|, - or if there are no more data items returns |NULL|. */ -void * -rb_t_next (struct rb_traverser *trav) -{ - struct rb_node *x; - - assert (trav != NULL); - - if (trav->rb_generation != trav->rb_table->rb_generation) - trav_refresh (trav); - - x = trav->rb_node; - if (x == NULL) - { - return rb_t_first (trav, trav->rb_table); - } - else if (x->rb_link[1] != NULL) - { - assert (trav->rb_height < RB_MAX_HEIGHT); - trav->rb_stack[trav->rb_height++] = x; - x = x->rb_link[1]; - - while (x->rb_link[0] != NULL) - { - assert (trav->rb_height < RB_MAX_HEIGHT); - trav->rb_stack[trav->rb_height++] = x; - x = x->rb_link[0]; - } - } - else - { - struct rb_node *y; - - do - { - if (trav->rb_height == 0) - { - trav->rb_node = NULL; - return NULL; - } - - y = x; - x = trav->rb_stack[--trav->rb_height]; - } - while (y == x->rb_link[1]); - } - trav->rb_node = x; - - return x->rb_data; -} - -/* Returns the previous data item in inorder - within the tree being traversed with |trav|, - or if there are no more data items returns |NULL|. */ -void * -rb_t_prev (struct rb_traverser *trav) -{ - struct rb_node *x; - - assert (trav != NULL); - - if (trav->rb_generation != trav->rb_table->rb_generation) - trav_refresh (trav); - - x = trav->rb_node; - if (x == NULL) - { - return rb_t_last (trav, trav->rb_table); - } - else if (x->rb_link[0] != NULL) - { - assert (trav->rb_height < RB_MAX_HEIGHT); - trav->rb_stack[trav->rb_height++] = x; - x = x->rb_link[0]; - - while (x->rb_link[1] != NULL) - { - assert (trav->rb_height < RB_MAX_HEIGHT); - trav->rb_stack[trav->rb_height++] = x; - x = x->rb_link[1]; - } - } - else - { - struct rb_node *y; - - do - { - if (trav->rb_height == 0) - { - trav->rb_node = NULL; - return NULL; - } - - y = x; - x = trav->rb_stack[--trav->rb_height]; - } - while (y == x->rb_link[0]); - } - trav->rb_node = x; - - return x->rb_data; -} - -/* Returns |trav|'s current item. */ -void * -rb_t_cur (struct rb_traverser *trav) -{ - assert (trav != NULL); - - return trav->rb_node != NULL ? trav->rb_node->rb_data : NULL; -} - -/* Replaces the current item in |trav| by |new| and returns the item replaced. - |trav| must not have the null item selected. - The new item must not upset the ordering of the tree. */ -void * -rb_t_replace (struct rb_traverser *trav, void *new) -{ - void *old; - - assert (trav != NULL && trav->rb_node != NULL && new != NULL); - old = trav->rb_node->rb_data; - trav->rb_node->rb_data = new; - return old; -} - -/* Destroys |new| with |rb_destroy (new, destroy)|, - first setting right links of nodes in |stack| within |new| - to null pointers to avoid touching uninitialized data. */ -static void -copy_error_recovery (struct rb_node **stack, int height, - struct rb_table *new, rb_item_func *destroy) -{ - assert (stack != NULL && height >= 0 && new != NULL); - - for (; height > 2; height -= 2) - stack[height - 1]->rb_link[1] = NULL; - rb_destroy (new, destroy); -} - -/* Copies |org| to a newly created tree, which is returned. - If |copy != NULL|, each data item in |org| is first passed to |copy|, - and the return values are inserted into the tree, - with |NULL| return values taken as indications of failure. - On failure, destroys the partially created new tree, - applying |destroy|, if non-null, to each item in the new tree so far, - and returns |NULL|. - If |allocator != NULL|, it is used for allocation in the new tree. - Otherwise, the same allocator used for |org| is used. */ -struct rb_table * -rb_copy (const struct rb_table *org, rb_copy_func *copy, - rb_item_func *destroy, struct libavl_allocator *allocator) -{ - struct rb_node *stack[2 * (RB_MAX_HEIGHT + 1)]; - int height = 0; - - struct rb_table *new; - const struct rb_node *x; - struct rb_node *y; - - assert (org != NULL); - new = rb_create (org->rb_compare, org->rb_param, - allocator != NULL ? allocator : org->rb_alloc); - if (new == NULL) - return NULL; - new->rb_count = org->rb_count; - if (new->rb_count == 0) - return new; - - x = (const struct rb_node *) &org->rb_root; - y = (struct rb_node *) &new->rb_root; - for (;;) - { - while (x->rb_link[0] != NULL) - { - assert (height < 2 * (RB_MAX_HEIGHT + 1)); - - y->rb_link[0] = - new->rb_alloc->libavl_malloc (new->rb_alloc, - sizeof *y->rb_link[0]); - if (y->rb_link[0] == NULL) - { - if (y != (struct rb_node *) &new->rb_root) - { - y->rb_data = NULL; - y->rb_link[1] = NULL; - } - - copy_error_recovery (stack, height, new, destroy); - return NULL; - } - - stack[height++] = (struct rb_node *) x; - stack[height++] = y; - x = x->rb_link[0]; - y = y->rb_link[0]; - } - y->rb_link[0] = NULL; - - for (;;) - { - y->rb_color = x->rb_color; - if (copy == NULL) - y->rb_data = x->rb_data; - else - { - y->rb_data = copy (x->rb_data, org->rb_param); - if (y->rb_data == NULL) - { - y->rb_link[1] = NULL; - copy_error_recovery (stack, height, new, destroy); - return NULL; - } - } - - if (x->rb_link[1] != NULL) - { - y->rb_link[1] = - new->rb_alloc->libavl_malloc (new->rb_alloc, - sizeof *y->rb_link[1]); - if (y->rb_link[1] == NULL) - { - copy_error_recovery (stack, height, new, destroy); - return NULL; - } - - x = x->rb_link[1]; - y = y->rb_link[1]; - break; - } - else - y->rb_link[1] = NULL; - - if (height <= 2) - return new; - - y = stack[--height]; - x = stack[--height]; - } - } -} - -/* Frees storage allocated for |tree|. - If |destroy != NULL|, applies it to each data item in inorder. */ -void -rb_destroy (struct rb_table *tree, rb_item_func *destroy) -{ - struct rb_node *p, *q; - - assert (tree != NULL); - - for (p = tree->rb_root; p != NULL; p = q) - if (p->rb_link[0] == NULL) - { - q = p->rb_link[1]; - if (destroy != NULL && p->rb_data != NULL) - destroy (p->rb_data, tree->rb_param); - tree->rb_alloc->libavl_free (tree->rb_alloc, p); - } - else - { - q = p->rb_link[0]; - p->rb_link[0] = q->rb_link[1]; - q->rb_link[1] = p; - } - - tree->rb_alloc->libavl_free (tree->rb_alloc, tree); -} - -/* Allocates |size| bytes of space using |malloc()|. - Returns a null pointer if allocation fails. */ -void * -rb_malloc (struct libavl_allocator *allocator, size_t size) -{ - assert (allocator != NULL && size > 0); - return malloc (size); -} - -/* Frees |block|. */ -void -rb_free (struct libavl_allocator *allocator, void *block) -{ - assert (allocator != NULL && block != NULL); - free (block); -} - -/* Default memory allocator that uses |malloc()| and |free()|. */ -struct libavl_allocator rb_allocator_default = - { - rb_malloc, - rb_free - }; - -#undef NDEBUG -#include - -/* Asserts that |rb_insert()| succeeds at inserting |item| into |table|. */ -void -(rb_assert_insert) (struct rb_table *table, void *item) -{ - void **p = rb_probe (table, item); - assert (p != NULL && *p == item); -} - -/* Asserts that |rb_delete()| really removes |item| from |table|, - and returns the removed item. */ -void * -(rb_assert_delete) (struct rb_table *table, void *item) -{ - void *p = rb_delete (table, item); - assert (p != NULL); - return p; -} - diff -Nru osm2pgsql-0.82.0/rb.h osm2pgsql-0.88.1/rb.h --- osm2pgsql-0.82.0/rb.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/rb.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,122 +0,0 @@ -/* Produced by texiweb from libavl.w. */ - -/* libavl - library for manipulation of binary trees. - Copyright (C) 1998-2002, 2004 Free Software Foundation, Inc. - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; either version 2 of the - License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. - - The author may be contacted at on the Internet, or - write to Ben Pfaff, Stanford University, Computer Science Dept., 353 - Serra Mall, Stanford CA 94305, USA. -*/ - -#ifndef RB_H -#define RB_H 1 - -#include - -/* Function types. */ -typedef int rb_comparison_func (const void *rb_a, const void *rb_b, - void *rb_param); -typedef void rb_item_func (void *rb_item, void *rb_param); -typedef void *rb_copy_func (void *rb_item, void *rb_param); - -#ifndef LIBAVL_ALLOCATOR -#define LIBAVL_ALLOCATOR -/* Memory allocator. */ -struct libavl_allocator - { - void *(*libavl_malloc) (struct libavl_allocator *, size_t libavl_size); - void (*libavl_free) (struct libavl_allocator *, void *libavl_block); - }; -#endif - -/* Default memory allocator. */ -extern struct libavl_allocator rb_allocator_default; -void *rb_malloc (struct libavl_allocator *, size_t); -void rb_free (struct libavl_allocator *, void *); - -/* Maximum RB height. */ -#ifndef RB_MAX_HEIGHT -#define RB_MAX_HEIGHT 48 -#endif - -/* Tree data structure. */ -struct rb_table - { - struct rb_node *rb_root; /* Tree's root. */ - rb_comparison_func *rb_compare; /* Comparison function. */ - void *rb_param; /* Extra argument to |rb_compare|. */ - struct libavl_allocator *rb_alloc; /* Memory allocator. */ - size_t rb_count; /* Number of items in tree. */ - unsigned long rb_generation; /* Generation number. */ - }; - -/* Color of a red-black node. */ -enum rb_color - { - RB_BLACK, /* Black. */ - RB_RED /* Red. */ - }; - -/* A red-black tree node. */ -struct rb_node - { - struct rb_node *rb_link[2]; /* Subtrees. */ - void *rb_data; /* Pointer to data. */ - unsigned char rb_color; /* Color. */ - }; - -/* RB traverser structure. */ -struct rb_traverser - { - struct rb_table *rb_table; /* Tree being traversed. */ - struct rb_node *rb_node; /* Current node in tree. */ - struct rb_node *rb_stack[RB_MAX_HEIGHT]; - /* All the nodes above |rb_node|. */ - size_t rb_height; /* Number of nodes in |rb_parent|. */ - unsigned long rb_generation; /* Generation number. */ - }; - -/* Table functions. */ -struct rb_table *rb_create (rb_comparison_func *, void *, - struct libavl_allocator *); -struct rb_table *rb_copy (const struct rb_table *, rb_copy_func *, - rb_item_func *, struct libavl_allocator *); -void rb_destroy (struct rb_table *, rb_item_func *); -void **rb_probe (struct rb_table *, void *); -void *rb_insert (struct rb_table *, void *); -void *rb_replace (struct rb_table *, void *); -void *rb_delete (struct rb_table *, const void *); -void *rb_find (const struct rb_table *, const void *); -void rb_assert_insert (struct rb_table *, void *); -void *rb_assert_delete (struct rb_table *, void *); - -#define rb_count(table) ((size_t) (table)->rb_count) - -/* Table traverser functions. */ -void rb_t_init (struct rb_traverser *, struct rb_table *); -void *rb_t_first (struct rb_traverser *, struct rb_table *); -void *rb_t_last (struct rb_traverser *, struct rb_table *); -void *rb_t_find (struct rb_traverser *, struct rb_table *, void *); -void *rb_t_insert (struct rb_traverser *, struct rb_table *, void *); -void *rb_t_copy (struct rb_traverser *, const struct rb_traverser *); -void *rb_t_next (struct rb_traverser *); -void *rb_t_prev (struct rb_traverser *); -void *rb_t_cur (struct rb_traverser *); -void *rb_t_replace (struct rb_traverser *, void *); - -#endif /* rb.h */ diff -Nru osm2pgsql-0.82.0/README osm2pgsql-0.88.1/README --- osm2pgsql-0.82.0/README 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/README 2015-08-15 05:28:11.000000000 +0000 @@ -1,301 +1,154 @@ -osm2pgsql -========= -Converts OSM planet.osm data to a PostgreSQL / PostGIS database suitable -for specific applications like rendering into map tiles by Mapnik -or geocoding with Nominatim. - -osm2pgsql currently supports two different database schemas -1) A database schema that is optimized for ease of rendering -by Mapnik. -2) A database schema that is optimized for geocoding with Nominatim, -emphasizing the spatially hierarchical organizations of objects. - -Both schemas were specifically optimized for the purpose they were -intended for and they may therefore be less suitable for other -general purpose processing. Nevertheless, the rendering schema -might be useful for other purposes as well, and has been used -for a variety of additionally purposes. - - -For a broader view of the whole map rendering tool chain see -http://wiki.openstreetmap.org/index.php/Mapnik -http://wiki.openstreetmap.org/index.php/Osm2pgsql -http://wiki.openstreetmap.org/index.php/Slippy_Map +# osm2pgsql # -You may find that the wiki pages are more up to date than this -readme and may include answers to issues not mentioned here. - -Any questions should be directed at the osm dev list -http://wiki.openstreetmap.org/index.php/Mailing_lists - -Features -======== -- Converts OSM files to a PostgreSQL DB -- Conversion of tags to columns is configurable in the style file -- Able to read .gz, .bz2, .pbf and .o5m files directly -- Can apply diffs to keep the database up to data -- Support the choice of output projection -- Configurable table names -- Gazetteer back-end for Nominatim - http://wiki.openstreetmap.org/wiki/Nominatim -- Support for hstore field type to store the complete set of tags in one database +osm2pgsql is a tool for loading OpenStreetMap data into a PostgreSQL / PostGIS +database suitable for applications like rendering into a map, geocoding with +Nominatim, or general analysis. + +## Features ## + +* Converts OSM files to a PostgreSQL DB +* Conversion of tags to columns is configurable in the style file +* Able to read .gz, .bz2, .pbf and .o5m files directly +* Can apply diffs to keep the database up to date +* Support the choice of output projection +* Configurable table names +* Gazetteer back-end for [Nominatim](http://wiki.openstreetmap.org/wiki/Nominatim) +* Support for hstore field type to store the complete set of tags in one database field if desired -Source code -=========== +## Installing ## + The latest source code is available in the OSM git repository on github and can be downloaded as follows: +```sh $ git clone git://github.com/openstreetmap/osm2pgsql.git +``` -Build requirements -================== -The code is written in C and C++ and relies on the libraries -below: -- libxml2 http://xmlsoft.org/ -- geos http://geos.refractions.net/ -- proj http://www.remotesensing.org/proj/ -- bzip2 http://www.bzip.org/ -- zlib http://www.zlib.net/ -- PostgreSQL http://www.postgresql.org/ -- PostGIS http://postgis.refractions.net/ - -To make use of the database generated by this tool you will -probably also want to install: -- Mapnik from http://mapnik.org/ +## Building ## +Osm2pgsql uses the [GNU Build System](http://www.gnu.org/software/automake/manual/html_node/GNU-Build-System.html) +to configure and build itself and requires +* [libxml2](http://xmlsoft.org/) +* [geos](http://geos.osgeo.org/) +* [proj](http://proj.osgeo.org/) +* [bzip2](http://www.bzip.org/) +* [zlib](http://www.zlib.net/) +* [Protocol Buffers](https://developers.google.com/protocol-buffers/) +* [PostgreSQL](http://www.postgresql.org/) client libraries +* [Lua](http://www.lua.org/) (Optional, used for [Lua tag transforms](docs/lua.md)) + +It also requires access to a database server running +[PostgreSQL](http://www.postgresql.org/) and [PostGIS](http://www.postgis.net/). -Building -======== Make sure you have installed the development packages for the libraries mentioned in the requirements section and a C and C++ compiler. -e.g. on Fedora: -# yum install geos-devel proj-devel postgresql-devel libxml2-devel bzip2-devel gcc-c++ +To install on a Debian or Ubuntu system, first install the prerequisites: + +```sh +sudo apt-get install autoconf automake libtool make g++ libboost-dev \ + libboost-system-dev libboost-filesystem-dev libboost-thread-dev libxml2-dev \ + libgeos-dev libgeos++-dev libpq-dev libbz2-dev libproj-dev \ + protobuf-c-compiler libprotobuf-c0-dev lua5.2 liblua5.2-dev +``` + +To install on a Fedora system, use + +```sh +sudo yum install gcc-c++ automake libtool boost-devel libxml2-devel \ + bzip2-devel postgresql-devel geos-devel proj-devel lua-devel \ + protobuf-c-devel +``` + +To install on a FreeBSD system, use + +```sh +pkg install devel/git devel/autoconf devel/automake devel/gmake devel/libtool \ + textproc/libxml2 graphics/geos graphics/proj databases/postgresql94-client \ + devel/boost-libs devel/protobuf-c lang/lua52 devel/pkgconf +``` + +Then you should be able to bootstrap the build system: + + ./autogen.sh + +And then run the standard GNU build install: + + ./configure && make && make install + +Please see `./configure --help` for more options on how to control the build +process. + +On FreeBSD instead bootstrap and then run -on Debian: -# aptitude install libxml2-dev libgeos-dev libgeos++-dev libpq-dev libbz2-dev libproj-dev protobuf-c-compiler libprotobuf-c0-dev autoconf automake libtool make g++ + LUA_LIB=`pkg-config --libs lua-5.2` ./configure && gmake && gmake install + +## Usage ## + +Osm2pgsql has one program, the executable itself, which has **43** command line +options. + +Before loading into a database, the database must be created and the PostGIS +and optionally hstore extensions must be loaded. A full guide to PostgreSQL +setup is beyond the scope of this readme, but with reasonably recent versions +of PostgreSQL and PostGIS this can be done with + +```sh +createdb gis +psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' +``` + +A basic invocation to load the data into the database ``gis`` for rendering would be + +```sh +osm2pgsql --create --database gis data.osm.pbf +``` + +This will load the data from ``data.osm.pbf`` into the ``planet_osm_point``, +``planet_osm_line``, ``planet_osm_roads``, and ``planet_osm_polygon`` tables. + +When importing a large amount of data such as the complete planet, a typical +command line would be + +```sh +osm2pgsql -c -d gis --slim -C \ + --flat-nodes planet-latest.osm.pbf +``` +where +* ```` is 24000 on machines with 32GiB or more RAM + or about 75% of memory in MiB on machines with less +* ```` is a location where a 24GiB file can be saved. + +The databases from either of these commands can be used immediately by +[Mapnik](http://mapnik.org/) for rendering maps with standard tools like +[renderd/mod_tile](https://github.com/openstreetmap/mod_tile), +[TileMill](https://www.mapbox.com/tilemill/), [Nik4](https://github.com/Zverik/Nik4), +among others. It can also be used for [spatial analysis](docs/analysis.md) or +[shapefile exports](docs/export.md). + +[Additional documentation is available on writing command lines](docs/usage.md). + +## Alternate backends ## + +In addition to the standard [pgsql](docs/pgsql.md) backend designed for +rendering there is also the [gazetteer](docs/gazetteer.md) database for +geocoding, principally with [Nominatim](http://www.nominatim.org/), and the +null backend for testing. For flexibility a new [multi](docs/multi.md) +backend is also avialable which allows the configuration of custom +postgres tables instead of those provided in the pgsql backend. + +Any questions should be directed at the osm dev list +http://wiki.openstreetmap.org/index.php/Mailing_lists -On most Unix-like systems the program can be compiled by -running './autogen.sh && ./configure && make'. +## Contributing ## -Operation -========= -You must create a PostgreSQL user and a database with the -PostGIS functions enabled. This requires access as the -database administrator, normally the 'postgres' user. - -The default name for this database is 'gis' but this may -be changed by using the osm2pgsql --database option. - -If the matches the unix user id running the import -and rendering then this allows the PostgreSQL 'ident sameuser' -authentication to be used which avoids the need to enter a -password when accessing the database. This is setup by default -on many Unix installs but does not work on Windows (due to the -lack of unix sockets). - -Some example commands are given below but you may find -this wiki page has more up to data information: -http://wiki.openstreetmap.org/wiki/Mapnik/PostGIS - -$ sudo -u postgres createuser -$ sudo -u postgres createdb -E UTF8 -O -$ sudo -u postgres createlang plpgsql - -Adding the PostGIS extensions. Note the location of the -files may vary. - -$ sudo -u postgres psql < /usr/share/postgresql/8.4/contrib/postgis-1.5/postgis.sql -$ sudo -u postgres psql < /usr/share/postgresql/8.4/contrib/postgis-1.5/spatial_ref_sys.sql - -Next we need to give the access to update the postgis -meta-data tables - -$ sudo -u postgres psql -d -c "ALTER TABLE geometry_columns OWNER TO " -$ sudo -u postgres psql -d -c "ALTER TABLE spatial_ref_sys OWNER TO " - -The 900913 is not normally included with PostGIS. To add it you -should run: - -$ sudo psql -u postgres psql -d -f 900913.sql - -If you want to use hstore support then you will also need to enable the PostgreSQL -hstore-new extension. - -$ sudo -u postgres psql < /usr/share/postgresql/8.4/contrib/hstore.sql - -On PostgreSQL 9.1 and above, you can install it by running "CREATE EXTENSION hstore;" -in your database. - -Now you can run osm2pgsql to import the OSM data. -This will perform the following actions: - -1) Osm2pgsql connects to database and creates the following 4 tables -when used with the default output back-end (pgsql): - - planet_osm_point - - planet_osm_line - - planet_osm_roads - - planet_osm_polygon -The prefix "planet_osm" can be changed with the --prefix option, -the above is the default. - -If you are using --slim mode, it will create the following additional 3 tables: - - planet_osm_nodes - - planet_osm_ways - - planet_osm_rels - - -2) Runs an XML parser on the input file (typically planet.osm) - and processes the nodes, ways and relations. - -3) If a node has a tag declared in the style file then it is - added to planet_osm_point. If it has no such tag then - the position is noted, but not added to the database. - -4) Ways are read in converted into WKT geometries by using the - positions of the nodes read in earlier. If the tags on the way - are listed in the style file then the way will be written into - the line or roads tables. - -5) If the way has one or more tags marked as 'polygon' and - forms a closed ring then it will be added to the planet_osm_polygon - table. - -6) The relations are parsed. Osm2pgsql has special handling for a - limited number of types: multipolygon, route, boundary - The code will build the appropriate geometries by referencing the - members and outputting these into the database. - -7) Indexes are added to speed up the queries by Mapnik. - -Tuning PostgreSQL -================= - -For an efficient operation of PostgreSQL you will need to tune the config -parameters of PostgreSQL from its default values. These are set in the -config file at /etc/postgresql/8.4/main/postgresql.conf - -The values you need to set will depend on the hardware you have available, -but you will likely need to increase the values for the following parameters: - -- shared_buffers -- checkpoint_segments -- work_mem -- maintenance_work_mem -- effective_cache_size - - -A quick note on projections -=========================== - -Depending on the command-line switches you can select which projection you -want the database in. You have three choices: - -4326: The standard lat/long coordinates -900913: The spherical Mercator projection, used by TileCache, Google Earth etc. -3395: The legacy (broken) WGS84 Mercator projection - -Depending on what you're using one or the other is appropriate. The default -Mapnik style (osm.xml) assumes that the data is stored in 900913 and this -is the default for osm2pgsql. - -Combining the -v and -h switches will tell about the exact definitions of -the projections. - -In case you want to use some completely different projection there is the -E -option. It will initialize the projection as +init=epsg:. This allows -you to use any projection recognized by proj4, which is useful if you want -to make a map in a different projection. These projections are usually -defined in /usr/share/proj/epsg. - -Database Access Examples -======================== -If you wish to access the data from the database then the -queries below should give you some hints. Note that these -examples all use the 'latlong' projection which is not the -default. - -$ psql gis -gis=> \d - List of relations - Schema | Name | Type | Owner ---------+--------------------+-------+---------- -... - public | planet_osm_line | table | jburgess - public | planet_osm_point | table | jburgess - public | planet_osm_polygon | table | jburgess - public | planet_osm_roads | table | jburgess -... - -gis=> \d planet_osm_line - Table "public.planet_osm_line" - Column | Type | Modifiers ------------+----------+----------- - osm_id | integer | - name | text | - place | text | - landuse | text | -... [ lots of stuff deleted ] ... - way | geometry | not null - z_order | integer | default 0 - - -Each of the tables contains a subset of the planet.osm file representing -a particular geometry type -- Point contains nodes which have interesting tags - e.g. place=city, name=London - -- Line contains ways with interesting tags - e.g. highway=motorway, ref=M25 - -- Polygon contains ways which form an enclosed area - e.g. landuse=reservoir - -The DB columns are used as follows: -- osm_id = the planet.osm ID of the node(point) or way(line,polygon) -- name, place, landuse, ... = the value of the given key, if present on -the node/way. If the tag is not present, the value is NULL. Only a -subset of all possible tags are stored in the DB. Only ones rendered in -the osm.xml are actually interesting to mapnik. -- way = PostGIS geometry describing the physical layout of the object. - - -Querying specific data requires knowlege of SQL and the OSM key/value -system, e.g. - -gis=> select osm_id,astext(way),name from planet_osm_point where amenity='cinema' limit 5; - osm_id | astext | name -----------+-------------------------------------------+-------------------- - 26236284 | POINT(-79.7160836579093 43.6802306464618) | - 26206699 | POINT(51.4051989797638 35.7066045032235) | Cinema Felestin - 26206700 | POINT(51.3994885141459 35.7058460359352) | Cinema Asr-e Jadid - 20979630 | POINT(151.225781789807 -33.8943079539886) | Paris Cinema - 20979684 | POINT(151.226855394904 -33.8946830511095) | Hoyts -(5 rows) - -Mapnik renders the data in each table by applying the rules in the -osm.xml file. - - -> How could I get e.g. all highways in a given bounding box? - -The 'way' column contains the geo info and is the one which you need to -use in your WHERE clause. e.g. - -gis=> select osm_id,highway,name from planet_osm_line where highway is not null and way && GeomFromText('POLYGON((0 52, 0.1 52, 0.1 52.1, 0 52.1, 0 52))',4326); - -osm_id | highway | name ----------+--------------+------------------ - 4273848 | unclassified | - 3977133 | trunk | to Royston (tbc) - 4004841 | trunk | - 4019198 | trunk | - 4019199 | trunk | - 4238966 | unclassified | +We welcome contributions to osm2pgsql. If you would like to report an issue, +please use the [issue tracker on GitHub](https://github.com/openstreetmap/osm2pgsql/issues). +More information can be found in [CONTRIBUTING.md](CONTRIBUTING.md). -See the Postgis docs for details, e.g. -http://postgis.refractions.net/docs/ch04.html +General queries can be sent to the tile-serving@ or dev@ +[mailing lists](http://wiki.openstreetmap.org/wiki/Mailing_lists). diff -Nru osm2pgsql-0.82.0/README.md osm2pgsql-0.88.1/README.md --- osm2pgsql-0.82.0/README.md 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/README.md 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,154 @@ +# osm2pgsql # + +osm2pgsql is a tool for loading OpenStreetMap data into a PostgreSQL / PostGIS +database suitable for applications like rendering into a map, geocoding with +Nominatim, or general analysis. + +## Features ## + +* Converts OSM files to a PostgreSQL DB +* Conversion of tags to columns is configurable in the style file +* Able to read .gz, .bz2, .pbf and .o5m files directly +* Can apply diffs to keep the database up to date +* Support the choice of output projection +* Configurable table names +* Gazetteer back-end for [Nominatim](http://wiki.openstreetmap.org/wiki/Nominatim) +* Support for hstore field type to store the complete set of tags in one database + field if desired + +## Installing ## + +The latest source code is available in the OSM git repository on github +and can be downloaded as follows: + +```sh +$ git clone git://github.com/openstreetmap/osm2pgsql.git +``` + +## Building ## + +Osm2pgsql uses the [GNU Build System](http://www.gnu.org/software/automake/manual/html_node/GNU-Build-System.html) +to configure and build itself and requires + +* [libxml2](http://xmlsoft.org/) +* [geos](http://geos.osgeo.org/) +* [proj](http://proj.osgeo.org/) +* [bzip2](http://www.bzip.org/) +* [zlib](http://www.zlib.net/) +* [Protocol Buffers](https://developers.google.com/protocol-buffers/) +* [PostgreSQL](http://www.postgresql.org/) client libraries +* [Lua](http://www.lua.org/) (Optional, used for [Lua tag transforms](docs/lua.md)) + +It also requires access to a database server running +[PostgreSQL](http://www.postgresql.org/) and [PostGIS](http://www.postgis.net/). + +Make sure you have installed the development packages for the +libraries mentioned in the requirements section and a C and C++ +compiler. + +To install on a Debian or Ubuntu system, first install the prerequisites: + +```sh +sudo apt-get install autoconf automake libtool make g++ libboost-dev \ + libboost-system-dev libboost-filesystem-dev libboost-thread-dev libxml2-dev \ + libgeos-dev libgeos++-dev libpq-dev libbz2-dev libproj-dev \ + protobuf-c-compiler libprotobuf-c0-dev lua5.2 liblua5.2-dev +``` + +To install on a Fedora system, use + +```sh +sudo yum install gcc-c++ automake libtool boost-devel libxml2-devel \ + bzip2-devel postgresql-devel geos-devel proj-devel lua-devel \ + protobuf-c-devel +``` + +To install on a FreeBSD system, use + +```sh +pkg install devel/git devel/autoconf devel/automake devel/gmake devel/libtool \ + textproc/libxml2 graphics/geos graphics/proj databases/postgresql94-client \ + devel/boost-libs devel/protobuf-c lang/lua52 devel/pkgconf +``` + +Then you should be able to bootstrap the build system: + + ./autogen.sh + +And then run the standard GNU build install: + + ./configure && make && make install + +Please see `./configure --help` for more options on how to control the build +process. + +On FreeBSD instead bootstrap and then run + + LUA_LIB=`pkg-config --libs lua-5.2` ./configure && gmake && gmake install + +## Usage ## + +Osm2pgsql has one program, the executable itself, which has **43** command line +options. + +Before loading into a database, the database must be created and the PostGIS +and optionally hstore extensions must be loaded. A full guide to PostgreSQL +setup is beyond the scope of this readme, but with reasonably recent versions +of PostgreSQL and PostGIS this can be done with + +```sh +createdb gis +psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' +``` + +A basic invocation to load the data into the database ``gis`` for rendering would be + +```sh +osm2pgsql --create --database gis data.osm.pbf +``` + +This will load the data from ``data.osm.pbf`` into the ``planet_osm_point``, +``planet_osm_line``, ``planet_osm_roads``, and ``planet_osm_polygon`` tables. + +When importing a large amount of data such as the complete planet, a typical +command line would be + +```sh +osm2pgsql -c -d gis --slim -C \ + --flat-nodes planet-latest.osm.pbf +``` +where +* ```` is 24000 on machines with 32GiB or more RAM + or about 75% of memory in MiB on machines with less +* ```` is a location where a 24GiB file can be saved. + +The databases from either of these commands can be used immediately by +[Mapnik](http://mapnik.org/) for rendering maps with standard tools like +[renderd/mod_tile](https://github.com/openstreetmap/mod_tile), +[TileMill](https://www.mapbox.com/tilemill/), [Nik4](https://github.com/Zverik/Nik4), +among others. It can also be used for [spatial analysis](docs/analysis.md) or +[shapefile exports](docs/export.md). + +[Additional documentation is available on writing command lines](docs/usage.md). + +## Alternate backends ## + +In addition to the standard [pgsql](docs/pgsql.md) backend designed for +rendering there is also the [gazetteer](docs/gazetteer.md) database for +geocoding, principally with [Nominatim](http://www.nominatim.org/), and the +null backend for testing. For flexibility a new [multi](docs/multi.md) +backend is also avialable which allows the configuration of custom +postgres tables instead of those provided in the pgsql backend. + +Any questions should be directed at the osm dev list +http://wiki.openstreetmap.org/index.php/Mailing_lists + +## Contributing ## + +We welcome contributions to osm2pgsql. If you would like to report an issue, +please use the [issue tracker on GitHub](https://github.com/openstreetmap/osm2pgsql/issues). + +More information can be found in [CONTRIBUTING.md](CONTRIBUTING.md). + +General queries can be sent to the tile-serving@ or dev@ +[mailing lists](http://wiki.openstreetmap.org/wiki/Mailing_lists). diff -Nru osm2pgsql-0.82.0/reprojection.c osm2pgsql-0.88.1/reprojection.c --- osm2pgsql-0.82.0/reprojection.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/reprojection.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,208 +0,0 @@ -/* reprojection.c - * - * Convert OSM coordinates to another coordinate system for - * the database (usually convert lat/lon to Spherical Mercator - * so Mapnik doesn't have to). - */ - -#include -#include -#include -#include -#include - -#include "reprojection.h" - -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif - -/** must match expire.tiles.c */ -#define EARTH_CIRCUMFERENCE 40075016.68 - -/** The projection of the source data. Always lat/lon (EPSG:4326). */ -static projPJ pj_source = NULL; - -/** The target projection (used in the PostGIS tables). Controlled by the -l/-M/-m/-E options. */ -static projPJ pj_target = NULL; - -/** The projection used for tiles. Currently this is fixed to be Spherical - * Mercator. You will usually have tiles in the same projection as used - * for PostGIS, but it is theoretically possible to have your PostGIS data - * in, say, lat/lon but still create tiles in Spherical Mercator. - */ -static projPJ pj_tile = NULL; - -static int Proj; - -const struct Projection_Info Projection_Info[] = { - [PROJ_LATLONG] = { - .descr = "Latlong", - .proj4text = "+init=epsg:4326", - .srs = 4326, - .option = "-l" }, - [PROJ_MERC] = { - .descr = "WGS84 Mercator", - .proj4text = "+proj=merc +datum=WGS84 +k=1.0 +units=m +over +no_defs", - .srs = 3395, - .option = "-M" }, - [PROJ_SPHERE_MERC] = { - .descr = "Spherical Mercator", - .proj4text = "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs", - .srs = 900913, - .option = "-m" } -}; -static struct Projection_Info custom_projection; - -/** defined in expire-tiles.c; depends on the zoom level selected for expiry. */ -extern int map_width; - -/* Positive numbers refer the to the table above, negative numbers are - assumed to refer to EPSG codes and it uses the proj4 to find those. */ -void project_init(int proj) -{ - char buffer[32]; - Proj = proj; - - /* hard-code the source projection to be lat/lon, since OSM XML always - * has coordinates in degrees. */ - pj_source = pj_init_plus("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"); - - /* hard-code the tile projection to be spherical mercator always. - * theoretically this could be made selectable but not all projections - * lend themselves well to making tiles; non-spherical mercator tiles - * are uncharted waters in OSM. */ - pj_tile = pj_init_plus(Projection_Info[PROJ_SPHERE_MERC].proj4text); - - /* now set the target projection - the only one which is really variable */ - if (proj >= 0 && proj < PROJ_COUNT) - { - pj_target = pj_init_plus(Projection_Info[proj].proj4text); - } - else if (proj < 0) - { - if (snprintf(buffer, sizeof(buffer), "+init=epsg:%d", -proj ) >= (int)sizeof(buffer)) - { - fprintf(stderr, "Buffer overflow computing proj4 initialisation string\n"); - exit(1); - } - pj_target = pj_init_plus(buffer); - if (!pj_target) - { - fprintf (stderr, "Couldn't read EPSG definition (do you have /usr/share/proj/epsg?)\n"); - exit(1); - } - } - - if (!pj_source || !pj_target || !pj_tile) - { - fprintf(stderr, "Projection code failed to initialise\n"); - exit(1); - } - - if (proj >= 0) - return; - - custom_projection.srs = -proj; - custom_projection.proj4text = pj_get_def(pj_target, 0); - if (snprintf(buffer, sizeof(buffer), "EPSG:%d", -proj) >= (int)sizeof(buffer)) - { - fprintf(stderr, "Buffer overflow computing projection description\n"); - exit(1); - } - custom_projection.descr = strdup(buffer); - custom_projection.option = "-E"; - return; -} - -void project_exit(void) -{ - pj_free(pj_source); - pj_source = NULL; - pj_free(pj_target); - pj_target = NULL; -} - -struct Projection_Info const *project_getprojinfo(void) -{ - if( Proj >= 0 ) - return &Projection_Info[Proj]; - else - return &custom_projection; -} - -void reproject(double *lat, double *lon) -{ - double x[1], y[1], z[1]; - - /** Caution: This section is only correct if the source projection is lat/lon; - * so even if it looks like pj_source was just a variable, things break if - * pj_source is something else than lat/lon. */ - - if (Proj == PROJ_LATLONG) - return; - - if (Proj == PROJ_SPHERE_MERC) - { - /* The latitude co-ordinate is clipped at slightly larger than the 900913 'world' - * extent of +-85.0511 degrees to ensure that the points appear just outside the - * edge of the map. */ - - if (*lat > 85.07) - *lat = 85.07; - if (*lat < -85.07) - *lat = -85.07; - - *lat = log(tan(M_PI/4.0 + (*lat) * DEG_TO_RAD / 2.0)) * EARTH_CIRCUMFERENCE/(M_PI*2); - *lon = (*lon) * EARTH_CIRCUMFERENCE / 360.0; - return; - } - - x[0] = *lon * DEG_TO_RAD; - y[0] = *lat * DEG_TO_RAD; - z[0] = 0; - - /** end of "caution" section. */ - - pj_transform(pj_source, pj_target, 1, 1, x, y, z); - - *lat = y[0]; - *lon = x[0]; -} - -/** - * Converts from (target) coordinates to tile coordinates. - * - * The zoom level for the coordinates is implicitly given in the global - * variable map_width. - */ -void coords_to_tile(double *tilex, double *tiley, double lon, double lat) -{ - double x[1], y[1], z[1]; - - x[0] = lon; - y[0] = lat; - z[0] = 0; - - if (Proj == PROJ_LATLONG) - { - x[0] *= DEG_TO_RAD; - y[0] *= DEG_TO_RAD; - } - - /* since pj_tile is always spherical merc, don't bother doing anything if - * destination proj is the same. */ - - if (Proj != PROJ_SPHERE_MERC) - { - pj_transform(pj_target, pj_tile, 1, 1, x, y, z); - /** FIXME: pj_transform could fail if coordinates are outside +/- 85 degrees latitude */ - } - - /* if ever pj_tile were allowed to be PROJ_LATLONG then results would have to - * be divided by DEG_TO_RAD here. */ - - *tilex = map_width * (0.5 + x[0] / EARTH_CIRCUMFERENCE); - *tiley = map_width * (0.5 - y[0] / EARTH_CIRCUMFERENCE); -} - diff -Nru osm2pgsql-0.82.0/reprojection.cpp osm2pgsql-0.88.1/reprojection.cpp --- osm2pgsql-0.82.0/reprojection.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/reprojection.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,205 @@ +/* reprojection.c + * + * Convert OSM coordinates to another coordinate system for + * the database (usually convert lat/lon to Spherical Mercator + * so Mapnik doesn't have to). + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "reprojection.hpp" + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +/** must match expire.tiles.c */ +#define EARTH_CIRCUMFERENCE 40075016.68 + +Projection_Info::Projection_Info(const char *descr_, const char *proj4text_, int srs_, const char *option_) + : descr(descr_), proj4text(proj4text_), srs(srs_), option(option_) { +} + +namespace { + +const struct Projection_Info Projection_Infos[] = { + /*PROJ_LATLONG*/ Projection_Info( + /*descr */ "Latlong", + /*proj4text*/ "+init=epsg:4326", + /*srs */ 4326, + /*option */ "-l" ), + /*PROJ_SPHERE_MERC*/ Projection_Info( + /*descr */ "Spherical Mercator", + /*proj4text*/ "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs", + /*srs */ 900913, + /*option */ "-m" ) +}; + +} // anonymous namespace + +/* Positive numbers refer the to the table above, negative numbers are + assumed to refer to EPSG codes and it uses the proj4 to find those. */ +reprojection::reprojection(int proj) + : Proj(proj), pj_source(NULL), pj_target(NULL), pj_tile(NULL), + custom_projection(NULL) +{ + char buffer[32]; + + /* hard-code the source projection to be lat/lon, since OSM XML always + * has coordinates in degrees. */ + pj_source = pj_init_plus("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"); + + /* hard-code the tile projection to be spherical mercator always. + * theoretically this could be made selectable but not all projections + * lend themselves well to making tiles; non-spherical mercator tiles + * are uncharted waters in OSM. */ + pj_tile = pj_init_plus(Projection_Infos[PROJ_SPHERE_MERC].proj4text); + + /* now set the target projection - the only one which is really variable */ + if (proj >= 0 && proj < PROJ_COUNT) + { + pj_target = pj_init_plus(Projection_Infos[proj].proj4text); + } + else if (proj < 0) + { + if (snprintf(buffer, sizeof(buffer), "+init=epsg:%d", -proj ) >= (int)sizeof(buffer)) + { + fprintf(stderr, "Buffer overflow computing proj4 initialisation string\n"); + exit(1); + } + pj_target = pj_init_plus(buffer); + if (!pj_target) + { + fprintf (stderr, "Couldn't read EPSG definition (do you have /usr/share/proj/epsg?)\n"); + exit(1); + } + } + + if (!pj_source || !pj_target || !pj_tile) + { + fprintf(stderr, "Projection code failed to initialise\n"); + exit(1); + } + + if (proj >= 0) + return; + + if (snprintf(buffer, sizeof(buffer), "EPSG:%d", -proj) >= (int)sizeof(buffer)) + { + fprintf(stderr, "Buffer overflow computing projection description\n"); + exit(1); + } + custom_projection = new Projection_Info( + strdup(buffer), + pj_get_def(pj_target, 0), + -proj, "-E"); +} + +reprojection::~reprojection() +{ + pj_free(pj_source); + pj_source = NULL; + pj_free(pj_target); + pj_target = NULL; + pj_free(pj_tile); + pj_tile = NULL; + + if (custom_projection != NULL) { + delete custom_projection; + } +} + +struct Projection_Info const *reprojection::project_getprojinfo(void) +{ + if( Proj >= 0 ) + return &Projection_Infos[Proj]; + else + return custom_projection; +} + +void reprojection::reproject(double *lat, double *lon) +{ + double x[1], y[1], z[1]; + + /** Caution: This section is only correct if the source projection is lat/lon; + * so even if it looks like pj_source was just a variable, things break if + * pj_source is something else than lat/lon. */ + + if (Proj == PROJ_LATLONG) + return; + + if (Proj == PROJ_SPHERE_MERC) + { + /* The latitude co-ordinate is clipped at slightly larger than the 900913 'world' + * extent of +-85.0511 degrees to ensure that the points appear just outside the + * edge of the map. */ + + if (*lat > 85.07) + *lat = 85.07; + if (*lat < -85.07) + *lat = -85.07; + + *lat = log(tan(M_PI/4.0 + (*lat) * DEG_TO_RAD / 2.0)) * EARTH_CIRCUMFERENCE/(M_PI*2); + *lon = (*lon) * EARTH_CIRCUMFERENCE / 360.0; + return; + } + + x[0] = *lon * DEG_TO_RAD; + y[0] = *lat * DEG_TO_RAD; + z[0] = 0; + + /** end of "caution" section. */ + + pj_transform(pj_source, pj_target, 1, 1, x, y, z); + + *lat = y[0]; + *lon = x[0]; +} + +/** + * Converts from (target) coordinates to tile coordinates. + * + * The zoom level for the coordinates is explicitly given in the + * variable map_width. + */ +void reprojection::coords_to_tile(double *tilex, double *tiley, double lon, double lat, + int map_width) +{ + double x[1], y[1], z[1]; + + x[0] = lon; + y[0] = lat; + z[0] = 0; + + if (Proj == PROJ_LATLONG) + { + x[0] *= DEG_TO_RAD; + y[0] *= DEG_TO_RAD; + } + + /* since pj_tile is always spherical merc, don't bother doing anything if + * destination proj is the same. */ + + if (Proj != PROJ_SPHERE_MERC) + { + pj_transform(pj_target, pj_tile, 1, 1, x, y, z); + /** FIXME: pj_transform could fail if coordinates are outside +/- 85 degrees latitude */ + } + + /* if ever pj_tile were allowed to be PROJ_LATLONG then results would have to + * be divided by DEG_TO_RAD here. */ + + *tilex = map_width * (0.5 + x[0] / EARTH_CIRCUMFERENCE); + *tiley = map_width * (0.5 - y[0] / EARTH_CIRCUMFERENCE); +} + +int reprojection::get_proj_id() const +{ + return Proj; +} diff -Nru osm2pgsql-0.82.0/reprojection.h osm2pgsql-0.88.1/reprojection.h --- osm2pgsql-0.82.0/reprojection.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/reprojection.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,27 +0,0 @@ -/* reprojection.h - * - * Convert OSM lattitude / longitude from degrees to mercator - * so that Mapnik does not have to project the data again - * - */ - -#ifndef REPROJECTION_H -#define REPROJECTION_H - -struct Projection_Info { - char *descr; - char *proj4text; - int srs; - char *option; -}; - -enum Projection { PROJ_LATLONG = 0, PROJ_MERC, PROJ_SPHERE_MERC, PROJ_COUNT }; -void project_init(int); -void project_exit(void); -struct Projection_Info const* project_getprojinfo(void); -void reproject(double *lat, double *lon); -void coords_to_tile(double *tilex, double *tiley, double lon, double lat); - -extern const struct Projection_Info Projection_Info[]; - -#endif diff -Nru osm2pgsql-0.82.0/reprojection.hpp osm2pgsql-0.88.1/reprojection.hpp --- osm2pgsql-0.82.0/reprojection.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/reprojection.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,53 @@ +/* reprojection.h + * + * Convert OSM lattitude / longitude from degrees to mercator + * so that Mapnik does not have to project the data again + * + */ + +#ifndef REPROJECTION_H +#define REPROJECTION_H + +#include + +struct Projection_Info { + Projection_Info(const char *descr_, const char *proj4text_, int srs_, const char *option_); + + const char *descr; + const char *proj4text; + const int srs; + const char *option; +}; + +enum Projection { PROJ_LATLONG = 0, PROJ_SPHERE_MERC, PROJ_COUNT }; + +struct reprojection : public boost::noncopyable +{ + explicit reprojection(int proj); + ~reprojection(); + + struct Projection_Info const* project_getprojinfo(void); + void reproject(double *lat, double *lon); + void coords_to_tile(double *tilex, double *tiley, double lon, double lat, int map_width); + int get_proj_id() const; + +private: + int Proj; + + /** The projection of the source data. Always lat/lon (EPSG:4326). */ + void *pj_source; + + /** The target projection (used in the PostGIS tables). Controlled by the -l/-M/-m/-E options. */ + void *pj_target; + + /** The projection used for tiles. Currently this is fixed to be Spherical + * Mercator. You will usually have tiles in the same projection as used + * for PostGIS, but it is theoretically possible to have your PostGIS data + * in, say, lat/lon but still create tiles in Spherical Mercator. + */ + void *pj_tile; + + struct Projection_Info *custom_projection; +}; + +#endif diff -Nru osm2pgsql-0.82.0/sample.multi.json osm2pgsql-0.88.1/sample.multi.json --- osm2pgsql-0.82.0/sample.multi.json 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/sample.multi.json 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,16 @@ + [ + { + "name": "building", + "type": "polygon", + "tagtransform": "building.lua", + "tagtransform-node-function": "nodes_proc", + "tagtransform-way-function": "ways_proc", + "tagtransform-relation-function": "rels_proc", + "tagtransform-relation-member-function": "rel_members_proc", + "tags": [ + {"name": "building", "type": "text"}, + {"name": "shop", "type": "text"}, + {"name": "amenity", "type": "text"} + ] + } + ] diff -Nru osm2pgsql-0.82.0/sanitizer.h osm2pgsql-0.88.1/sanitizer.h --- osm2pgsql-0.82.0/sanitizer.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/sanitizer.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ -#ifndef SANITIZER_H -#define SANITIZER_H - -#include -#include - -xmlTextReaderPtr sanitizerOpen(const char *name); - -#endif diff -Nru osm2pgsql-0.82.0/sanitizer.hpp osm2pgsql-0.88.1/sanitizer.hpp --- osm2pgsql-0.82.0/sanitizer.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/sanitizer.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,8 @@ +#ifndef SANITIZER_H +#define SANITIZER_H + +#include + +xmlTextReaderPtr sanitizerOpen(const char *name); + +#endif diff -Nru osm2pgsql-0.82.0/sprompt.c osm2pgsql-0.88.1/sprompt.c --- osm2pgsql-0.82.0/sprompt.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/sprompt.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,197 +0,0 @@ -/*------------------------------------------------------------------------- - * - * sprompt.c - * simple_prompt() routine - * - * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * - * IDENTIFICATION - * $PostgreSQL: pgsql/src/port/sprompt.c,v 1.18 2006/10/04 00:30:14 momjian Exp $ - * - *------------------------------------------------------------------------- - * - * PostgreSQL Database Management System - * (formerly known as Postgres, then as Postgres95) - * - * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group - * - * Portions Copyright (c) 1994, The Regents of the University of California - * - * Permission to use, copy, modify, and distribute this software and its - * documentation for any purpose, without fee, and without a written agreement - * is hereby granted, provided that the above copyright notice and this - * paragraph and the following two paragraphs appear in all copies. - * - * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR - * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING - * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS - * DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS - * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO - * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. - * - */ - - -/* - * simple_prompt - * - * Generalized function especially intended for reading in usernames and - * password interactively. Reads from /dev/tty or stdin/stderr. - * - * prompt: The prompt to print - * maxlen: How many characters to accept - * echo: Set to false if you want to hide what is entered (for passwords) - * - * Returns a malloc()'ed string with the input (w/o trailing newline). - */ - -#define DEVTTY "/dev/tty" - -#include -#include -#include -#include -#include - -#include - -#ifdef __MINGW_H -# include -#else -# define HAVE_TERMIOS_H -# include -#endif - -extern char *simple_prompt(const char *prompt, int maxlen, int echo); - -char * -simple_prompt(const char *prompt, int maxlen, int echo) -{ - int length; - char *destination; - FILE *termin, - *termout; - -#ifdef HAVE_TERMIOS_H - struct termios t_orig, - t; -#else -#ifdef WIN32 - HANDLE t = NULL; - LPDWORD t_orig = NULL; -#endif -#endif - - destination = (char *) malloc(maxlen + 1); - if (!destination) - return NULL; - - /* - * Do not try to collapse these into one "w+" mode file. Doesn't work on - * some platforms (eg, HPUX 10.20). - */ - termin = fopen(DEVTTY, "r"); - termout = fopen(DEVTTY, "w"); - if (!termin || !termout -#ifdef WIN32 - /* See DEVTTY comment for msys */ - || (getenv("OSTYPE") && strcmp(getenv("OSTYPE"), "msys") == 0) -#endif - ) - { - if (termin) - fclose(termin); - if (termout) - fclose(termout); - termin = stdin; - termout = stderr; - } - -#ifdef HAVE_TERMIOS_H - if (!echo) - { - tcgetattr(fileno(termin), &t); - t_orig = t; - t.c_lflag &= ~ECHO; - tcsetattr(fileno(termin), TCSAFLUSH, &t); - } -#else -#ifdef WIN32 - if (!echo) - { - /* get a new handle to turn echo off */ - t_orig = (LPDWORD) malloc(sizeof(DWORD)); - t = GetStdHandle(STD_INPUT_HANDLE); - - /* save the old configuration first */ - GetConsoleMode(t, t_orig); - - /* set to the new mode */ - SetConsoleMode(t, ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT); - } -#endif -#endif - - if (prompt) - { - fputs(prompt, termout); - fflush(termout); - } - - if (fgets(destination, maxlen + 1, termin) == NULL) - destination[0] = '\0'; - - length = strlen(destination); - if (length > 0 && destination[length - 1] != '\n') - { - /* eat rest of the line */ - char buf[128]; - int buflen; - - do - { - if (fgets(buf, sizeof(buf), termin) == NULL) - break; - buflen = strlen(buf); - } while (buflen > 0 && buf[buflen - 1] != '\n'); - } - - if (length > 0 && destination[length - 1] == '\n') - /* remove trailing newline */ - destination[length - 1] = '\0'; - -#ifdef HAVE_TERMIOS_H - if (!echo) - { - tcsetattr(fileno(termin), TCSAFLUSH, &t_orig); - fputs("\n", termout); - fflush(termout); - } -#else -#ifdef WIN32 - if (!echo) - { - /* reset to the original console mode */ - SetConsoleMode(t, *t_orig); - fputs("\n", termout); - fflush(termout); - free(t_orig); - } -#endif -#endif - - if (termin != stdin) - { - fclose(termin); - fclose(termout); - } - - return destination; -} diff -Nru osm2pgsql-0.82.0/sprompt.cpp osm2pgsql-0.88.1/sprompt.cpp --- osm2pgsql-0.82.0/sprompt.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/sprompt.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,196 @@ +/*------------------------------------------------------------------------- + * + * sprompt.c + * simple_prompt() routine + * + * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * $PostgreSQL: pgsql/src/port/sprompt.c,v 1.18 2006/10/04 00:30:14 momjian Exp $ + * + *------------------------------------------------------------------------- + * + * PostgreSQL Database Management System + * (formerly known as Postgres, then as Postgres95) + * + * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group + * + * Portions Copyright (c) 1994, The Regents of the University of California + * + * Permission to use, copy, modify, and distribute this software and its + * documentation for any purpose, without fee, and without a written agreement + * is hereby granted, provided that the above copyright notice and this + * paragraph and the following two paragraphs appear in all copies. + * + * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR + * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING + * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS + * DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS + * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO + * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + * + */ + + +/* + * simple_prompt + * + * Generalized function especially intended for reading in usernames and + * password interactively. Reads from /dev/tty or stdin/stderr. + * + * prompt: The prompt to print + * maxlen: How many characters to accept + * echo: Set to false if you want to hide what is entered (for passwords) + * + * Returns a malloc()'ed string with the input (w/o trailing newline). + */ + +#define DEVTTY "/dev/tty" + +#include +#include +#include +#include +#include + +#include + +#ifdef HAVE_TERMIOS_H +#include +#endif + +#ifdef _WIN32 +#include +#endif + +char * +simple_prompt(const char *prompt, int maxlen, int echo) +{ + int length; + char *destination; + FILE *termin, + *termout; + +#ifdef HAVE_TERMIOS_H + struct termios t_orig, + t; +#else +#ifdef _WIN32 + HANDLE t = NULL; + LPDWORD t_orig = NULL; +#endif +#endif + + destination = (char *) malloc(maxlen + 1); + if (!destination) + return NULL; + + /* + * Do not try to collapse these into one "w+" mode file. Doesn't work on + * some platforms (eg, HPUX 10.20). + */ + termin = fopen(DEVTTY, "r"); + termout = fopen(DEVTTY, "w"); + if (!termin || !termout +#ifdef _WIN32 + /* See DEVTTY comment for msys */ + || (getenv("OSTYPE") && strcmp(getenv("OSTYPE"), "msys") == 0) +#endif + ) + { + if (termin) + fclose(termin); + if (termout) + fclose(termout); + termin = stdin; + termout = stderr; + } + +#ifdef HAVE_TERMIOS_H + if (!echo) + { + tcgetattr(fileno(termin), &t); + t_orig = t; + t.c_lflag &= ~ECHO; + tcsetattr(fileno(termin), TCSAFLUSH, &t); + } +#else +#ifdef _WIN32 + if (!echo) + { + /* get a new handle to turn echo off */ + t_orig = (LPDWORD) malloc(sizeof(DWORD)); + t = GetStdHandle(STD_INPUT_HANDLE); + + /* save the old configuration first */ + GetConsoleMode(t, t_orig); + + /* set to the new mode */ + SetConsoleMode(t, ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT); + } +#endif +#endif + + if (prompt) + { + fputs(prompt, termout); + fflush(termout); + } + + if (fgets(destination, maxlen + 1, termin) == NULL) + destination[0] = '\0'; + + length = strlen(destination); + if (length > 0 && destination[length - 1] != '\n') + { + /* eat rest of the line */ + char buf[128]; + int buflen; + + do + { + if (fgets(buf, sizeof(buf), termin) == NULL) + break; + buflen = strlen(buf); + } while (buflen > 0 && buf[buflen - 1] != '\n'); + } + + if (length > 0 && destination[length - 1] == '\n') + /* remove trailing newline */ + destination[length - 1] = '\0'; + +#ifdef HAVE_TERMIOS_H + if (!echo) + { + tcsetattr(fileno(termin), TCSAFLUSH, &t_orig); + fputs("\n", termout); + fflush(termout); + } +#else +#ifdef _WIN32 + if (!echo) + { + /* reset to the original console mode */ + SetConsoleMode(t, *t_orig); + fputs("\n", termout); + fflush(termout); + free(t_orig); + } +#endif +#endif + + if (termin != stdin) + { + fclose(termin); + fclose(termout); + } + + return destination; +} diff -Nru osm2pgsql-0.82.0/sprompt.h osm2pgsql-0.88.1/sprompt.h --- osm2pgsql-0.82.0/sprompt.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/sprompt.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ -#ifndef SPROMPT_H -#define SPROMPT_H -char *simple_prompt(const char *prompt, int maxlen, int echo); -#endif diff -Nru osm2pgsql-0.82.0/sprompt.hpp osm2pgsql-0.88.1/sprompt.hpp --- osm2pgsql-0.82.0/sprompt.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/sprompt.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,4 @@ +#ifndef SPROMPT_H +#define SPROMPT_H +char *simple_prompt(const char *prompt, int maxlen, int echo); +#endif diff -Nru osm2pgsql-0.82.0/style.lua osm2pgsql-0.88.1/style.lua --- osm2pgsql-0.82.0/style.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/style.lua 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,211 @@ +-- For documentation of Lua tag transformations, see docs/lua.md. + +-- Objects with any of the following keys will be treated as polygon +polygon_keys = { 'building', 'landuse', 'amenity', 'harbour', 'historic', 'leisure', + 'man_made', 'military', 'natural', 'office', 'place', 'power', + 'public_transport', 'shop', 'sport', 'tourism', 'waterway', + 'wetland', 'water', 'aeroway' } + +-- Objects without any of the following keys will be deleted +generic_keys = {'access','addr:housename','addr:housenumber','addr:interpolation','admin_level','aerialway','aeroway','amenity','area','barrier', + 'bicycle','brand','bridge','boundary','building','capital','construction','covered','culvert','cutting','denomination','disused','ele', + 'embarkment','foot','generation:source','harbour','highway','historic','hours','intermittent','junction','landuse','layer','leisure','lock', + 'man_made','military','motor_car','name','natural','office','oneway','operator','place','poi','population','power','power_source','public_transport', + 'railway','ref','religion','route','service','shop','sport','surface','toll','tourism','tower:type', 'tracktype','tunnel','water','waterway', + 'wetland','width','wood','type'} + +-- The following keys will be deleted +delete_tags = { 'FIXME', 'note', 'source' } + +-- Array used to specify z_order per key/value combination. +-- Each element has the form {key, value, z_order, is_road}. +-- If is_road=1, the object will be added to planet_osm_roads. +zordering_tags = {{ 'railway', nil, 5, 1}, { 'boundary', 'administrative', 0, 1}, + { 'bridge', 'yes', 10, 0 }, { 'bridge', 'true', 10, 0 }, { 'bridge', 1, 10, 0 }, + { 'tunnel', 'yes', -10, 0}, { 'tunnel', 'true', -10, 0}, { 'tunnel', 1, -10, 0}, + { 'highway', 'minor', 3, 0}, { 'highway', 'road', 3, 0 }, { 'highway', 'unclassified', 3, 0 }, + { 'highway', 'residential', 3, 0 }, { 'highway', 'tertiary_link', 4, 0}, { 'highway', 'tertiary', 4, 0}, + { 'highway', 'secondary_link', 6, 1}, { 'highway', 'secondary', 6, 1}, + { 'highway', 'primary_link', 7, 1}, { 'highway', 'primary', 7, 1}, + { 'highway', 'trunk_link', 8, 1}, { 'highway', 'trunk', 8, 1}, + { 'highway', 'motorway_link', 9, 1}, { 'highway', 'motorway', 9, 1}, +} + +function add_z_order(keyvalues) + -- The default z_order is 0 + z_order = 0 + + -- Add the value of the layer key times 10 to z_order + if (keyvalues["layer"] ~= nil and tonumber(keyvalues["layer"])) then + z_order = 10*keyvalues["layer"] + end + + -- Increase or decrease z_order based on the specific key/value combination as specified in zordering_tags + for i,k in ipairs(zordering_tags) do + -- If the value in zordering_tags is specified, match key and value. Otherwise, match key only. + if ((k[2] and keyvalues[k[1]] == k[2]) or (k[2] == nil and keyvalues[k[1]] ~= nil)) then + -- If the fourth component of the element of zordering_tags is 1, add the object to planet_osm_roads + if (k[4] == 1) then + roads = 1 + end + z_order = z_order + k[3] + end + end + + -- Add z_order as key/value combination + keyvalues["z_order"] = z_order + + return keyvalues, roads +end + +-- Filtering on nodes, ways, and relations +function filter_tags_generic(keyvalues, numberofkeys) + filter = 0 -- Will object be filtered out? + + -- Filter out objects with 0 tags + if numberofkeys == 0 then + filter = 1 + return filter, keyvalues + end + + -- Delete tags listed in delete_tags + for i,k in ipairs(delete_tags) do + keyvalues[k] = nil + end + + -- Filter out objects that do not have any of the keys in generic_keys + tagcount = 0 + for k,v in pairs(keyvalues) do + for i, k2 in ipairs(generic_keys) do if k2 == k then tagcount = tagcount + 1; end end + end + if tagcount == 0 then + filter = 1 + end + + return filter, keyvalues +end + +-- Filtering on nodes +function filter_tags_node (keyvalues, numberofkeys) + return filter_tags_generic(keyvalues, numberofkeys) +end + +-- Filtering on relations +function filter_basic_tags_rel (keyvalues, numberofkeys) + -- Filter out objects that are filtered out by filter_tags_generic + filter, keyvalues = filter_tags_generic(keyvalues, numberofkeys) + if filter == 1 then + return filter, keyvalues + end + + -- Filter out all relations except route, multipolygon and boundary relations + if ((keyvalues["type"] ~= "route") and (keyvalues["type"] ~= "multipolygon") and (keyvalues["type"] ~= "boundary")) then + filter = 1 + return filter, keyvalues + end + + return filter, keyvalues +end + +-- Filtering on ways +function filter_tags_way (keyvalues, numberofkeys) + filter = 0 -- Will object be filtered out? + polygon = 0 -- Will object be treated as polygon? + roads = 0 -- Will object be added to planet_osm_roads? + + -- Filter out objects that are filtered out by filter_tags_generic + filter, keyvalues = filter_tags_generic(keyvalues, numberofkeys) + if filter == 1 then + return filter, keyvalues, polygon, roads + end + + -- Treat objects with a key in polygon_keys as polygon + for i,k in ipairs(polygon_keys) do + if keyvalues[k] then + polygon=1 + break + end + end + + -- Treat objects tagged as area=yes, area=1, or area=true as polygon, + -- and treat objects tagged as area=no, area=0, or area=false not as polygon + if ((keyvalues["area"] == "yes") or (keyvalues["area"] == "1") or (keyvalues["area"] == "true")) then + polygon = 1; + elseif ((keyvalues["area"] == "no") or (keyvalues["area"] == "0") or (keyvalues["area"] == "false")) then + polygon = 0; + end + + -- Add z_order key/value combination and determine if the object should also be added to planet_osm_roads + keyvalues, roads = add_z_order(keyvalues) + + return filter, keyvalues, polygon, roads +end + +function filter_tags_relation_member (keyvalues, keyvaluemembers, roles, membercount) + filter = 0 -- Will object be filtered out? + linestring = 0 -- Will object be treated as linestring? + polygon = 0 -- Will object be treated as polygon? + roads = 0 -- Will object be added to planet_osm_roads? + membersuperseded = {} + for i = 1, membercount do + membersuperseded[i] = 0 -- Will member be ignored when handling areas? + end + + type = keyvalues["type"] + + -- Remove type key + keyvalues["type"] = nil + + -- Relations with type=boundary are treated as linestring + if (type == "boundary") then + linestring = 1 + end + -- Relations with type=multipolygon and boundary=* are treated as linestring + if ((type == "multipolygon") and keyvalues["boundary"]) then + linestring = 1 + -- For multipolygons... + elseif (type == "multipolygon") then + -- Treat as polygon + polygon = 1 + polytagcount = 0; + -- Count the number of polygon tags of the object + for i,k in ipairs(polygon_keys) do + if keyvalues[k] then + polytagcount = polytagcount + 1 + end + end + -- If there are no polygon tags, add tags from all outer elements to the multipolygon itself + if (polytagcount == 0) then + for i = 1,membercount do + if (roles[i] == "outer") then + for k,v in pairs(keyvaluemembers[i]) do + keyvalues[k] = v + end + end + end + end + -- For any member of the multipolygon, set membersuperseded to 1 (i.e. don't deal with it as area as well), + -- except when the member has a key/value combination such that + -- 1) the key occurs in generic_keys + -- 2) the key/value combination is not also a key/value combination of the multipolygon itself + for i = 1,membercount do + superseded = 1 + for k,v in pairs(keyvaluemembers[i]) do + if ((keyvalues[k] == nil) or (keyvalues[k] ~= v)) then + for j,k2 in ipairs(generic_keys) do + if (k == k2) then + superseded = 0; + break + end + end + end + end + membersuperseded[i] = superseded + end + end + + -- Add z_order key/value combination and determine if the object should also be added to planet_osm_roads + keyvalues, roads = add_z_order(keyvalues) + + return filter, keyvalues, membersuperseded, linestring, polygon, roads +end diff -Nru osm2pgsql-0.82.0/table.cpp osm2pgsql-0.88.1/table.cpp --- osm2pgsql-0.82.0/table.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/table.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,566 @@ +#include "table.hpp" +#include "options.hpp" +#include "util.hpp" + +#include +#include +#include +#include +#include +#include + +using std::string; +typedef boost::format fmt; + +#define BUFFER_SEND_SIZE 1024 + + +table_t::table_t(const string& conninfo, const string& name, const string& type, const columns_t& columns, const hstores_t& hstore_columns, + const int srid, const bool append, const bool slim, const bool drop_temp, const int hstore_mode, + const bool enable_hstore_index, const boost::optional& table_space, const boost::optional& table_space_index) : + conninfo(conninfo), name(name), type(type), sql_conn(NULL), copyMode(false), srid((fmt("%1%") % srid).str()), + append(append), slim(slim), drop_temp(drop_temp), hstore_mode(hstore_mode), enable_hstore_index(enable_hstore_index), + columns(columns), hstore_columns(hstore_columns), table_space(table_space), table_space_index(table_space_index) +{ + //if we dont have any columns + if(columns.size() == 0) + throw std::runtime_error((fmt("No columns provided for table %1%") % name).str()); + + //nothing to copy to start with + buffer = ""; + + //we use these a lot, so instead of constantly allocating them we predefine these + single_fmt = fmt("%1%"); + point_fmt = fmt("POINT(%.15g %.15g)"); + del_fmt = fmt("DELETE FROM %1% WHERE osm_id = %2%"); +} + +table_t::table_t(const table_t& other): + conninfo(other.conninfo), name(other.name), type(other.type), sql_conn(NULL), copyMode(false), buffer(), srid(other.srid), + append(other.append), slim(other.slim), drop_temp(other.drop_temp), hstore_mode(other.hstore_mode), enable_hstore_index(other.enable_hstore_index), + columns(other.columns), hstore_columns(other.hstore_columns), copystr(other.copystr), table_space(other.table_space), + table_space_index(other.table_space_index), single_fmt(other.single_fmt), point_fmt(other.point_fmt), del_fmt(other.del_fmt) +{ + // if the other table has already started, then we want to execute + // the same stuff to get into the same state. but if it hasn't, then + // this would be premature. + if (other.sql_conn) { + connect(); + //let postgres cache this query as it will presumably happen a lot + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %1% WHERE osm_id = $1") % name).str()); + //start the copy + begin(); + pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr); + copyMode = true; + } +} + +table_t::~table_t() +{ + teardown(); +} + +std::string const& table_t::get_name() { + return name; +} + +void table_t::teardown() +{ + if(sql_conn != NULL) + { + PQfinish(sql_conn); + sql_conn = NULL; + } +} + +void table_t::begin() +{ + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "BEGIN"); +} + +void table_t::commit() +{ + stop_copy(); + fprintf(stderr, "Committing transaction for %s\n", name.c_str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "COMMIT"); +} + +void table_t::connect() +{ + //connect + PGconn* _conn = PQconnectdb(conninfo.c_str()); + if (PQstatus(_conn) != CONNECTION_OK) + throw std::runtime_error((fmt("Connection to database failed: %1%\n") % PQerrorMessage(_conn)).str()); + sql_conn = _conn; + //let commits happen faster by delaying when they actually occur + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "SET synchronous_commit TO off;"); +} + +void table_t::start() +{ + if(sql_conn) + throw std::runtime_error(name + " cannot start, its already started"); + + connect(); + fprintf(stderr, "Setting up table: %s\n", name.c_str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "SET client_min_messages = WARNING"); + //we are making a new table + if (!append) + { + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE IF EXISTS %1%") % name).str()); + }//we are checking in append mode that the srid you specified matches whats already there + else + { + boost::shared_ptr res = pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT srid FROM geometry_columns WHERE f_table_name='%1%';") % name).str()); + if (!((PQntuples(res.get()) == 1) && (PQnfields(res.get()) == 1))) + throw std::runtime_error((fmt("Problem reading geometry information for table %1% - does it exist?\n") % name).str()); + char* their_srid = PQgetvalue(res.get(), 0, 0); + if (srid.compare(their_srid) != 0) + throw std::runtime_error((fmt("SRID mismatch: cannot append to table %1% (SRID %2%) using selected SRID %3%\n") % name % their_srid % srid).str()); + } + + /* These _tmp tables can be left behind if we run out of disk space */ + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE IF EXISTS %1%_tmp") % name).str()); + + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, "RESET client_min_messages"); + begin(); + + //making a new table + if (!append) + { + //define the new table + string sql = (fmt("CREATE TABLE %1% (osm_id %2%,") % name % POSTGRES_OSMID_TYPE).str(); + + //first with the regular columns + for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column) + sql += (fmt("\"%1%\" %2%,") % column->first % column->second).str(); + + //then with the hstore columns + for(hstores_t::const_iterator hcolumn = hstore_columns.begin(); hcolumn != hstore_columns.end(); ++hcolumn) + sql += (fmt("\"%1%\" hstore,") % (*hcolumn)).str(); + + //add tags column + if (hstore_mode != HSTORE_NONE) + sql += "\"tags\" hstore)"; + //or remove the last ", " from the end + else + sql[sql.length() - 1] = ')'; + + //add the main table space + if (table_space) + sql += " TABLESPACE " + table_space.get(); + + //create the table + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, sql); + + //add some constraints + pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT AddGeometryColumn('%1%', 'way', %2%, '%3%', 2 )") % name % srid % type).str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ALTER TABLE %1% ALTER COLUMN way SET NOT NULL") % name).str()); + + //slim mode needs this to be able to apply diffs + if (slim && !drop_temp) { + sql = (fmt("CREATE INDEX %1%_pkey ON %2% USING BTREE (osm_id)") % name % name).str(); + if (table_space_index) + sql += " TABLESPACE " + table_space_index.get(); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, sql); + } + + }//appending + else { + //check the columns against those in the existing table + boost::shared_ptr res = pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT * FROM %1% LIMIT 0") % name).str()); + for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column) + { + if(PQfnumber(res.get(), ('"' + column->first + '"').c_str()) < 0) + { +#if 0 + throw std::runtime_error((fmt("Append failed. Column \"%1%\" is missing from \"%2%\"\n") % info.name % name).str()); +#else + fprintf(stderr, "%s", (fmt("Adding new column \"%1%\" to \"%2%\"\n") % column->first % name).str().c_str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ALTER TABLE %1% ADD COLUMN \"%2%\" %3%") % name % column->first % column->second).str()); +#endif + } + //Note: we do not verify the type or delete unused columns + } + + //TODO: check over hstore columns + + //TODO: change the type of the geometry column if needed - this can only change to a more permissive type + } + + //let postgres cache this query as it will presumably happen a lot + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("PREPARE get_wkt (" POSTGRES_OSMID_TYPE ") AS SELECT ST_AsText(way) FROM %1% WHERE osm_id = $1") % name).str()); + + //generate column list for COPY + string cols = "osm_id,"; + //first with the regular columns + for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column) + cols += (fmt("\"%1%\",") % column->first).str(); + + //then with the hstore columns + for(hstores_t::const_iterator hcolumn = hstore_columns.begin(); hcolumn != hstore_columns.end(); ++hcolumn) + cols += (fmt("\"%1%\",") % (*hcolumn)).str(); + + //add tags column and geom column + if (hstore_mode != HSTORE_NONE) + cols += "tags,way"; + //or just the geom column + else + cols += "way"; + + //get into copy mode + copystr = (fmt("COPY %1% (%2%) FROM STDIN") % name % cols).str(); + pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr); + copyMode = true; +} + +void table_t::stop() +{ + stop_copy(); + if (!append) + { + time_t start, end; + time(&start); + + fprintf(stderr, "Sorting data and creating indexes for %s\n", name.c_str()); + + // Special handling for empty geometries because geohash chokes on + // empty geometries on postgis 1.5. + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE TABLE %1%_tmp %2% AS SELECT * FROM %3% ORDER BY CASE WHEN ST_IsEmpty(way) THEN NULL ELSE ST_GeoHash(ST_Transform(ST_Envelope(way),4326),10) END") % name % (table_space ? "TABLESPACE " + table_space.get() : "") % name).str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("DROP TABLE %1%") % name).str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ALTER TABLE %1%_tmp RENAME TO %2%") % name % name).str()); + // Re-add constraints if on 1.x. 2.0 has typemod, and they automatically come with CREATE TABLE AS + pgsql_exec_simple(sql_conn, PGRES_TUPLES_OK, (fmt("SELECT CASE WHEN PostGIS_Lib_Version() LIKE '1.%%' THEN Populate_Geometry_Columns('%1%'::regclass) ELSE 1 END;") % name).str()); + fprintf(stderr, "Copying %s to cluster by geometry finished\n", name.c_str()); + fprintf(stderr, "Creating geometry index on %s\n", name.c_str()); + + // Use fillfactor 100 for un-updatable imports + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_index ON %2% USING GIST (way) %3% %4%") % name % name % + (slim && !drop_temp ? "" : "WITH (FILLFACTOR=100)") % + (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str()); + + /* slim mode needs this to be able to apply diffs */ + if (slim && !drop_temp) + { + fprintf(stderr, "Creating osm_id index on %s\n", name.c_str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_pkey ON %2% USING BTREE (osm_id) %3%") % name % name % + (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str()); + } + /* Create hstore index if selected */ + if (enable_hstore_index) { + fprintf(stderr, "Creating hstore indexes on %s\n", name.c_str()); + if (hstore_mode != HSTORE_NONE) { + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_tags_index ON %2% USING GIN (tags) %3%") % name % name % + (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str()); + } + for(size_t i = 0; i < hstore_columns.size(); ++i) { + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("CREATE INDEX %1%_hstore_%2%_index ON %3% USING GIN (\"%4%\") %5%") % name % i % name % hstore_columns[i] % + (table_space_index ? "TABLESPACE " + table_space_index.get() : "")).str()); + } + } + fprintf(stderr, "Creating indexes on %s finished\n", name.c_str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("GRANT SELECT ON %1% TO PUBLIC") % name).str()); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (fmt("ANALYZE %1%") % name).str()); + time(&end); + fprintf(stderr, "All indexes on %s created in %ds\n", name.c_str(), (int)(end - start)); + } + teardown(); + + fprintf(stderr, "Completed %s\n", name.c_str()); +} + +void table_t::stop_copy() +{ + PGresult* res; + int stop; + + //we werent copying anyway + if(!copyMode) + return; + //if there is stuff left over in the copy buffer send it offand copy it before we stop + else if(buffer.length() != 0) + { + pgsql_CopyData(name.c_str(), sql_conn, buffer); + buffer.clear(); + }; + + //stop the copy + stop = PQputCopyEnd(sql_conn, NULL); + if (stop != 1) + throw std::runtime_error((fmt("stop COPY_END for %1% failed: %2%\n") % name % PQerrorMessage(sql_conn)).str()); + + //get the result + res = PQgetResult(sql_conn); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + PQclear(res); + throw std::runtime_error((fmt("result COPY_END for %1% failed: %2%\n") % name % PQerrorMessage(sql_conn)).str()); + } + PQclear(res); + copyMode = false; +} + +void table_t::write_node(const osmid_t id, const taglist_t &tags, double lat, double lon) +{ + write_wkt(id, tags, (point_fmt % lon % lat).str().c_str()); +} + +void table_t::delete_row(const osmid_t id) +{ + stop_copy(); + pgsql_exec_simple(sql_conn, PGRES_COMMAND_OK, (del_fmt % name % id).str()); +} + +void table_t::write_wkt(const osmid_t id, const taglist_t &tags, const char *wkt) +{ + //add the osm id + buffer.append((single_fmt % id).str()); + buffer.push_back('\t'); + + // used to remember which columns have been written out already. + std::vector used; + + if (hstore_mode != HSTORE_NONE) + used.assign(tags.size(), false); + + //get the regular columns' values + write_columns(tags, buffer, hstore_mode == HSTORE_NORM?&used:NULL); + + //get the hstore columns' values + write_hstore_columns(tags, buffer); + + //get the key value pairs for the tags column + if (hstore_mode != HSTORE_NONE) + write_tags_column(tags, buffer, used); + + //give the wkt an srid + buffer.append("SRID="); + buffer.append(srid); + buffer.push_back(';'); + //add the wkt + buffer.append(wkt); + //we need \n because we are copying from stdin + buffer.push_back('\n'); + + //tell the db we are copying if for some reason we arent already + if (!copyMode) + { + pgsql_exec_simple(sql_conn, PGRES_COPY_IN, copystr); + copyMode = true; + } + + //send all the data to postgres + if(buffer.length() > BUFFER_SEND_SIZE) + { + pgsql_CopyData(name.c_str(), sql_conn, buffer); + buffer.clear(); + } +} + +void table_t::write_columns(const taglist_t &tags, string& values, std::vector *used) +{ + //for each column + for(columns_t::const_iterator column = columns.begin(); column != columns.end(); ++column) + { + int idx; + if ((idx = tags.indexof(column->first)) >= 0) + { + escape_type(tags[idx].value, column->second, values); + //remember we already used this one so we cant use again later in the hstore column + if (used) + (*used)[idx] = true; + } + else + values.append("\\N"); + values.push_back('\t'); + } +} + +void table_t::write_tags_column(const taglist_t &tags, std::string& values, + const std::vector &used) +{ + //iterate through the list of tags, first one is always null + bool added = false; + for (size_t i = 0; i < tags.size(); ++i) + { + const tag& xtag = tags[i]; + //skip z_order tag and keys which have their own column + if (used[i] || ("z_order" == xtag.key)) + continue; + + //hstore ASCII representation looks like "key"=>"value" + if(added) + values.push_back(','); + escape4hstore(xtag.key.c_str(), values); + values.append("=>"); + escape4hstore(xtag.value.c_str(), values); + + //we did at least one so we need commas from here on out + added = true; + } + + //finish the hstore column by placing a TAB into the data stream + values.push_back('\t'); +} + +/* write an hstore column to the database */ +void table_t::write_hstore_columns(const taglist_t &tags, std::string& values) +{ + //iterate over all configured hstore columns in the options + for(hstores_t::const_iterator hstore_column = hstore_columns.begin(); hstore_column != hstore_columns.end(); ++hstore_column) + { + bool added = false; + + //iterate through the list of tags, first one is always null + for (taglist_t::const_iterator xtags = tags.begin(); xtags != tags.end(); ++xtags) + { + //check if the tag's key starts with the name of the hstore column + if(xtags->key.compare(0, hstore_column->size(), *hstore_column) == 0) + { + //generate the short key name, somehow pointer arithmetic works against the key string... + const char* shortkey = xtags->key.c_str() + hstore_column->size(); + + //and pack the shortkey with its value into the hstore + //hstore ASCII representation looks like "key"=>"value" + if(added) + values.push_back(','); + escape4hstore(shortkey, values); + values.append("=>"); + escape4hstore(xtags->value.c_str(), values); + + //we did at least one so we need commas from here on out + added = true; + } + } + + //if you found not matching tags write a NULL + if(!added) + values.append("\\N"); + + //finish the column off with a tab + values.push_back('\t'); + } +} + +//create an escaped version of the string for hstore table insert +void table_t::escape4hstore(const char *src, string& dst) +{ + dst.push_back('"'); + for (size_t i = 0; i < strlen(src); ++i) { + switch (src[i]) { + case '\\': + dst.append("\\\\\\\\"); + break; + case '"': + dst.append("\\\\\""); + break; + case '\t': + dst.append("\\\t"); + break; + case '\r': + dst.append("\\\r"); + break; + case '\n': + dst.append("\\\n"); + break; + default: + dst.push_back(src[i]); + break; + } + } + dst.push_back('"'); +} + +/* Escape data appropriate to the type */ +void table_t::escape_type(const string &value, const string &type, string& dst) { + + // For integers we take the first number, or the average if it's a-b + if (type == "int4") { + int from, to; + int items = sscanf(value.c_str(), "%d-%d", &from, &to); + if (items == 1) + dst.append((single_fmt % from).str()); + else if (items == 2) + dst.append((single_fmt % ((from + to) / 2)).str()); + else + dst.append("\\N"); + } + /* try to "repair" real values as follows: + * assume "," to be a decimal mark which need to be replaced by "." + * like int4 take the first number, or the average if it's a-b + * assume SI unit (meters) + * convert feet to meters (1 foot = 0.3048 meters) + * reject anything else + */ + else if (type == "real") + { + string escaped(value); + std::replace(escaped.begin(), escaped.end(), ',', '.'); + + float from, to; + int items = sscanf(escaped.c_str(), "%f-%f", &from, &to); + if (items == 1) + { + if (escaped.size() > 1 && escaped.substr(escaped.size() - 2).compare("ft") == 0) + from *= 0.3048; + dst.append((single_fmt % from).str()); + } + else if (items == 2) + { + if (escaped.size() > 1 && escaped.substr(escaped.size() - 2).compare("ft") == 0) + { + from *= 0.3048; + to *= 0.3048; + } + dst.append((single_fmt % ((from + to) / 2)).str()); + } + else + dst.append("\\N"); + }//just a string + else + escape(value, dst); +} + +boost::shared_ptr table_t::get_wkt_reader(const osmid_t id) +{ + //cant get wkt using the prepared statement without stopping the copy first + stop_copy(); + + char const *paramValues[1]; + char tmp[16]; + snprintf(tmp, sizeof(tmp), "%" PRIdOSMID, id); + paramValues[0] = tmp; + + //the prepared statement get_wkt will behave differently depending on the sql_conn + //each table has its own sql_connection with the get_way referring to the appropriate table + PGresult* res = pgsql_execPrepared(sql_conn, "get_wkt", 1, (const char * const *)paramValues, PGRES_TUPLES_OK); + return boost::shared_ptr(new wkt_reader(res)); +} + +table_t::wkt_reader::wkt_reader(PGresult* result):result(result), current(0) +{ + count = PQntuples(result); +} + +table_t::wkt_reader::~wkt_reader() +{ + PQclear(result); +} + +const char* table_t::wkt_reader::get_next() +{ + if(current < count) + return PQgetvalue(result, current++, 0); + return NULL; +} + +size_t table_t::wkt_reader::get_count() const +{ + return count; +} + +void table_t::wkt_reader::reset() +{ + //NOTE: PQgetvalue doc doesn't say if you can call it multiple times with the same row col + current = 0; +} diff -Nru osm2pgsql-0.82.0/table.hpp osm2pgsql-0.88.1/table.hpp --- osm2pgsql-0.82.0/table.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/table.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,91 @@ +#ifndef TABLE_H +#define TABLE_H + +#include "pgsql.hpp" +#include "osmtypes.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +typedef std::vector hstores_t; +typedef std::vector > columns_t; + +class table_t +{ + public: + table_t(const std::string& conninfo, const std::string& name, const std::string& type, const columns_t& columns, const hstores_t& hstore_columns, const int srid, + const bool append, const bool slim, const bool droptemp, const int hstore_mode, const bool enable_hstore_index, + const boost::optional& table_space, const boost::optional& table_space_index); + table_t(const table_t& other); + ~table_t(); + + void start(); + void stop(); + + void begin(); + void commit(); + + void write_wkt(const osmid_t id, const taglist_t &tags, const char *wkt); + void write_node(const osmid_t id, const taglist_t &tags, double lat, double lon); + void delete_row(const osmid_t id); + + std::string const& get_name(); + + //interface from retrieving well known text geometry from the table + struct wkt_reader + { + friend class table_t; + public: + virtual ~wkt_reader(); + const char* get_next(); + size_t get_count() const; + void reset(); + private: + wkt_reader(PGresult* result); + PGresult* result; + size_t count; + size_t current; + }; + boost::shared_ptr get_wkt_reader(const osmid_t id); + + protected: + void connect(); + void stop_copy(); + void teardown(); + + void write_columns(const taglist_t &tags, std::string& values, std::vector *used); + void write_tags_column(const taglist_t &tags, std::string& values, + const std::vector &used); + void write_hstore_columns(const taglist_t &tags, std::string& values); + + void escape4hstore(const char *src, std::string& dst); + void escape_type(const std::string &value, const std::string &type, std::string& dst); + + std::string conninfo; + std::string name; + std::string type; + pg_conn *sql_conn; + bool copyMode; + std::string buffer; + std::string srid; + bool append; + bool slim; + bool drop_temp; + int hstore_mode; + bool enable_hstore_index; + columns_t columns; + hstores_t hstore_columns; + std::string copystr; + boost::optional table_space; + boost::optional table_space_index; + + boost::format single_fmt, point_fmt, del_fmt; +}; + +#endif diff -Nru osm2pgsql-0.82.0/taginfo.cpp osm2pgsql-0.88.1/taginfo.cpp --- osm2pgsql-0.82.0/taginfo.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/taginfo.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,215 @@ +#include "taginfo_impl.hpp" +#include "table.hpp" +#include "util.hpp" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef strtok_r + #define strtok_r strtok_s +#endif +#endif + +/* NOTE: section below for flags genuinely is static and + * constant, so there's no need to hoist this into a per + * class variable. It doesn't get modified, so it's safe + * to share across threads and its lifetime is the whole + * program. + */ +struct flagsname { + flagsname(const char *name_, int flag_) + : name(name_), flag(flag_) { + } + const char *name; + int flag; +}; + +static const flagsname tagflags[] = { + flagsname("polygon", FLAG_POLYGON), + flagsname("linear", FLAG_LINEAR), + flagsname("nocache", FLAG_NOCACHE), + flagsname("delete", FLAG_DELETE), + flagsname("phstore", FLAG_PHSTORE) +}; +#define NUM_FLAGS ((signed)(sizeof(tagflags) / sizeof(tagflags[0]))) + +taginfo::taginfo() + : name(), type(), flags(0) { +} + +taginfo::taginfo(const taginfo &other) + : name(other.name), type(other.type), + flags(other.flags) { +} + +export_list::export_list() + : num_tables(0), exportList() { +} + +void export_list::add(enum OsmType id, const taginfo &info) { + std::vector &infos = get(id); + infos.push_back(info); +} + +std::vector &export_list::get(enum OsmType id) { + if (id >= num_tables) { + exportList.resize(id+1); + num_tables = id + 1; + } + return exportList[id]; +} + +const std::vector &export_list::get(enum OsmType id) const { + // this fakes as if we have infinite taginfo vectors, but + // means we don't actually have anything allocated unless + // the info object has been assigned. + static const std::vector empty; + + if (id < num_tables) { + return exportList[id]; + } else { + return empty; + } +} + +columns_t export_list::normal_columns(enum OsmType id) const { + columns_t columns; + const std::vector &infos = get(id); + for(std::vector::const_iterator info = infos.begin(); info != infos.end(); ++info) + { + if( info->flags & FLAG_DELETE ) + continue; + if( (info->flags & FLAG_PHSTORE ) == FLAG_PHSTORE) + continue; + columns.push_back(std::pair(info->name, info->type)); + } + return columns; +} + +int parse_tag_flags(const char *flags_, int lineno) { + int temp_flags = 0; + char *str = NULL, *saveptr = NULL; + int i = 0; + + // yuck! but strtok requires a non-const char * pointer, and i'm fairly sure it + // doesn't actually modify the string. + char *flags = const_cast(flags_); + + //split the flags column on commas and keep track of which flags you've seen in a bit mask + for(str = strtok_r(flags, ",\r\n", &saveptr); str != NULL; str = strtok_r(NULL, ",\r\n", &saveptr)) + { + for( i=0; iadd(OSMTYPE_NODE, temp); + kept = true; + } + + //keep this tag info if it applies to ways + if( strstr( osmtype, "way" ) ) + { + exlist->add(OSMTYPE_WAY, temp); + kept = true; + } + + //do we really want to completely quit on an unusable line? + if( !kept ) + { + throw std::runtime_error((boost::format("Weird style line %1%:%2%") + % filename % lineno).str()); + } + num_read++; + } + + + if (ferror(in)) { + throw std::runtime_error((boost::format("%1%: %2%") + % filename % strerror(errno)).str()); + } + if (num_read == 0) { + throw std::runtime_error("Unable to parse any valid columns from " + "the style file. Aborting."); + } + fclose(in); + return enable_way_area; +} diff -Nru osm2pgsql-0.82.0/taginfo.hpp osm2pgsql-0.88.1/taginfo.hpp --- osm2pgsql-0.82.0/taginfo.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/taginfo.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,10 @@ +#ifndef TAGINFO_HPP +#define TAGINFO_HPP + +/* Table columns, representing key= tags */ +struct taginfo; + +/* list of exported tags */ +struct export_list; + +#endif /* TAGINFO_HPP */ diff -Nru osm2pgsql-0.82.0/taginfo_impl.hpp osm2pgsql-0.88.1/taginfo_impl.hpp --- osm2pgsql-0.82.0/taginfo_impl.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/taginfo_impl.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,50 @@ +#ifndef TAGINFO_IMPL_HPP +#define TAGINFO_IMPL_HPP + +#include "taginfo.hpp" +#include "osmtypes.hpp" +#include +#include +#include + +#define FLAG_POLYGON 1 /* For polygon table */ +#define FLAG_LINEAR 2 /* For lines table */ +#define FLAG_NOCACHE 4 /* Optimisation: don't bother remembering this one */ +#define FLAG_DELETE 8 /* These tags should be simply deleted on sight */ +#define FLAG_PHSTORE 17 /* polygons without own column but listed in hstore this implies FLAG_POLYGON */ + +struct taginfo { + taginfo(); + taginfo(const taginfo &); + + std::string name, type; + int flags; +}; + +struct export_list { + export_list(); + + void add(enum OsmType id, const taginfo &info); + std::vector &get(enum OsmType id); + const std::vector &get(enum OsmType id) const; + + std::vector > normal_columns(enum OsmType id) const; + + int num_tables; + std::vector > exportList; /* Indexed by enum OsmType */ +}; + +/* Parse a comma or whitespace delimited list of tags to apply to + * a style file entry, returning the OR-ed set of flags. + */ +int parse_tag_flags(const char *flags, int lineno); + +/* Parse an osm2pgsql "pgsql" backend format style file, putting + * the results in the `exlist` argument. + * + * Returns 1 if the 'way_area' column should (implicitly) exist, or + * 0 if it should be suppressed. + */ +int read_style_file( const std::string &filename, export_list *exlist ); + +#endif /* TAGINFO_IMPL_HPP */ diff -Nru osm2pgsql-0.82.0/tagtransform.cpp osm2pgsql-0.88.1/tagtransform.cpp --- osm2pgsql-0.82.0/tagtransform.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tagtransform.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,674 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "tagtransform.hpp" +#include "options.hpp" +#include "config.h" +#include "wildcmp.hpp" +#include "taginfo_impl.hpp" + +#ifdef HAVE_LUA +extern "C" { + #include + #include +} +#endif + + +static const struct { + int offset; + const char *highway; + int roads; +} layers[] = { + { 1, "proposed", 0 }, + { 2, "construction", 0 }, + { 10, "steps", 0 }, + { 10, "cycleway", 0 }, + { 10, "bridleway", 0 }, + { 10, "footway", 0 }, + { 10, "path", 0 }, + { 11, "track", 0 }, + { 15, "service", 0 }, + + { 24, "tertiary_link", 0 }, + { 25, "secondary_link",1 }, + { 27, "primary_link", 1 }, + { 28, "trunk_link", 1 }, + { 29, "motorway_link", 1 }, + + { 30, "raceway", 0 }, + { 31, "pedestrian", 0 }, + { 32, "living_street", 0 }, + { 33, "road", 0 }, + { 33, "unclassified", 0 }, + { 33, "residential", 0 }, + { 34, "tertiary", 0 }, + { 36, "secondary", 1 }, + { 37, "primary", 1 }, + { 38, "trunk", 1 }, + { 39, "motorway", 1 } +}; + +static const unsigned int nLayers = (sizeof(layers)/sizeof(*layers)); + +namespace { +void add_z_order(taglist_t &tags, int *roads) +{ + const std::string *layer = tags.get("layer"); + const std::string *highway = tags.get("highway"); + bool bridge = tags.get_bool("bridge", false); + bool tunnel = tags.get_bool("tunnel", false); + const std::string *railway = tags.get("railway"); + const std::string *boundary = tags.get("boundary"); + + int z_order = 0; + + int l = layer ? strtol(layer->c_str(), NULL, 10) : 0; + z_order = 100 * l; + *roads = 0; + + if (highway) { + for (unsigned i = 0; i < nLayers; i++) { + if (!strcmp(layers[i].highway, highway->c_str())) { + z_order += layers[i].offset; + *roads = layers[i].roads; + break; + } + } + } + + if (railway && !railway->empty()) { + z_order += 35; + *roads = 1; + } + /* Administrative boundaries are rendered at low zooms so we prefer to use the roads table */ + if (boundary && *boundary == "administrative") + *roads = 1; + + if (bridge) + z_order += 100; + + if (tunnel) + z_order -= 100; + + char z[13]; + snprintf(z, sizeof(z), "%d", z_order); + tags.push_back(tag("z_order", z)); +} + +unsigned int c_filter_rel_member_tags(const taglist_t &rel_tags, + const multitaglist_t &member_tags, const rolelist_t &member_roles, + int *member_superseeded, int *make_boundary, int *make_polygon, int *roads, + const export_list &exlist, taglist_t &out_tags, bool allow_typeless) +{ + //if it has a relation figure out what kind it is + const std::string *type = rel_tags.get("type"); + bool is_route = false, is_boundary = false, is_multipolygon = false; + if (type) + { + //what kind of relation is it + if (*type == "route") + is_route = true; + else if (*type == "boundary") + is_boundary = true; + else if (*type == "multipolygon") + is_multipolygon = true; + }//you didnt have a type and it was required + else if (!allow_typeless) + { + return 1; + } + + /* Clone tags from relation */ + for (taglist_t::const_iterator it = rel_tags.begin(); it != rel_tags.end(); ++it) { + //copy the name tag as "route_name" + if (is_route && (it->key == "name")) + out_tags.push_dedupe(tag("route_name", it->value)); + //copy all other tags except for "type" + if (it->key != "type") + out_tags.push_dedupe(*it); + } + + if (is_route) { + const std::string *netw = rel_tags.get("network"); + int networknr = -1; + + if (netw != NULL) { + const std::string *state = rel_tags.get("state"); + std::string statetype("yes"); + if (state) { + if (*state == "alternate") + statetype = "alternate"; + else if (*state == "connection") + statetype = "connection"; + } + if (*netw == "lcn") { + networknr = 10; + out_tags.push_dedupe(tag("lcn", statetype)); + } else if (*netw == "rcn") { + networknr = 11; + out_tags.push_dedupe(tag("rcn", statetype)); + } else if (*netw == "ncn") { + networknr = 12; + out_tags.push_dedupe(tag("ncn", statetype)); + } else if (*netw == "lwn") { + networknr = 20; + out_tags.push_dedupe(tag("lwn", statetype)); + } else if (*netw == "rwn") { + networknr = 21; + out_tags.push_dedupe(tag("rwn", statetype)); + } else if (*netw == "nwn") { + networknr = 22; + out_tags.push_dedupe(tag("nwn", statetype)); + } + } + + const std::string *prefcol = rel_tags.get("preferred_color"); + if (prefcol != NULL && prefcol->size() == 1) { + if ((*prefcol)[0] == '0' || (*prefcol)[0] == '1' + || (*prefcol)[0] == '2' || (*prefcol)[0] == '3' + || (*prefcol)[0] == '4') { + out_tags.push_dedupe(tag("route_pref_color", *prefcol)); + } else { + out_tags.push_dedupe(tag("route_pref_color", "0")); + } + } else { + out_tags.push_dedupe(tag("route_pref_color", "0")); + } + + const std::string *relref = rel_tags.get("ref"); + if (relref != NULL ) { + if (networknr == 10) { + out_tags.push_dedupe(tag("lcn_ref", *relref)); + } else if (networknr == 11) { + out_tags.push_dedupe(tag("rcn_ref", *relref)); + } else if (networknr == 12) { + out_tags.push_dedupe(tag("ncn_ref", *relref)); + } else if (networknr == 20) { + out_tags.push_dedupe(tag("lwn_ref", *relref)); + } else if (networknr == 21) { + out_tags.push_dedupe(tag("rwn_ref", *relref)); + } else if (networknr == 22) { + out_tags.push_dedupe(tag("nwn_ref", *relref)); + } + } + } else if (is_boundary) { + /* Boundaries will get converted into multiple geometries: + - Linear features will end up in the line and roads tables (useful for admin boundaries) + - Polygon features also go into the polygon table (useful for national_forests) + The edges of the polygon also get treated as linear fetaures allowing these to be rendered seperately. */ + *make_boundary = 1; + } else if (is_multipolygon && out_tags.contains("boundary")) { + /* Treat type=multipolygon exactly like type=boundary if it has a boundary tag. */ + *make_boundary = 1; + } else if (is_multipolygon) { + *make_polygon = 1; + + /* Collect a list of polygon-like tags, these are used later to + identify if an inner rings looks like it should be rendered separately */ + taglist_t poly_tags; + for (taglist_t::const_iterator it = out_tags.begin(); it != out_tags.end(); ++it) { + if (it->key == "area") { + poly_tags.push_back(*it); + } else { + const std::vector &infos = exlist.get(OSMTYPE_WAY); + for (std::vector::const_iterator info = infos.begin(); + info != infos.end(); ++info) { + if (info->name == it->key) { + if (info->flags & FLAG_POLYGON) { + poly_tags.push_back(*it); + } + break; + } + } + } + } + + /* Copy the tags from the outer way(s) if the relation is untagged (with + * respect to tags that influence its polygon nature. Tags like name or fixme should be fine*/ + if (poly_tags.empty()) { + int first_outerway = 1; + for (size_t i = 0; i < member_tags.size(); i++) { + if (member_roles[i] && *(member_roles[i]) == "inner") + continue; + + /* insert all tags of the first outerway to the potential list of copied tags. */ + if (first_outerway) { + for (taglist_t::const_iterator it = member_tags[i].begin(); + it != member_tags[i].end(); ++it) + poly_tags.push_back(*it); + } else { + /* Check if all of the tags in the list of potential tags are present on this way, + otherwise remove from the list of potential tags. Tags need to be present on + all outer ways to be copied over to the relation */ + taglist_t::iterator it = poly_tags.begin(); + while (it != poly_tags.end()) { + if (!member_tags[i].contains(it->key)) + /* This tag is not present on all member outer ways, so don't copy it over to relation */ + it = poly_tags.erase(it); + else + ++it; + } + } + first_outerway = 0; + } + /* Copy the list identified outer way tags over to the relation */ + for (taglist_t::const_iterator it = poly_tags.begin(); it != poly_tags.end(); ++it) + out_tags.push_dedupe(*it); + + /* We need to re-check and only keep polygon tags in the list of polytags */ + // TODO what is that for? The list is cleared just below. + taglist_t::iterator q = poly_tags.begin(); + const std::vector &infos = exlist.get(OSMTYPE_WAY); + while (q != poly_tags.end()) { + bool contains_tag = false; + for (std::vector::const_iterator info = infos.begin(); + info != infos.end(); ++info) { + if (info->name == q->key) { + if (info->flags & FLAG_POLYGON) { + contains_tag = true; + } + break; + } + } + + if (contains_tag) + ++q; + else + q = poly_tags.erase(q); + } + } + } else if(!allow_typeless) { + /* Unknown type, just exit */ + out_tags.clear(); + return 1; + } + + if (out_tags.empty()) { + return 1; + } + + /* If we are creating a multipolygon then we + mark each member so that we can skip them during iterate_ways + but only if the polygon-tags look the same as the outer ring */ + if (make_polygon) { + for (size_t i = 0; i < member_tags.size(); i++) { + member_superseeded[i] = 1; + for (taglist_t::const_iterator p = member_tags[i].begin(); + p != member_tags[i].end(); ++p) { + const std::string *v = out_tags.get(p->key); + if (!v || *v != p->value) { + /* z_order and osm_ are automatically generated tags, so ignore them */ + if ((p->key != "z_order") && (p->key != "osm_user") && + (p->key != "osm_version") && (p->key != "osm_uid") && + (p->key != "osm_changeset") && (p->key != "osm_timestamp")) { + member_superseeded[i] = 0; + break; + } + } + } + } + } + + add_z_order(out_tags, roads); + + return 0; +} +} // anonymous namespace + +#ifdef HAVE_LUA +unsigned tagtransform::lua_filter_rel_member_tags(const taglist_t &rel_tags, + const multitaglist_t &member_tags, const rolelist_t &member_roles, + int *member_superseeded, int *make_boundary, int *make_polygon, int *roads, + taglist_t &out_tags) +{ + lua_getglobal(L, m_rel_mem_func.c_str()); + + lua_newtable(L); /* relations key value table */ + + for (taglist_t::const_iterator it = rel_tags.begin(); it != rel_tags.end(); ++it) { + lua_pushstring(L, it->key.c_str()); + lua_pushstring(L, it->value.c_str()); + lua_rawset(L, -3); + } + + lua_newtable(L); /* member tags table */ + + int idx = 1; + for (multitaglist_t::const_iterator list = member_tags.begin(); + list != member_tags.end(); ++list) { + lua_pushnumber(L, idx++); + lua_newtable(L); /* member key value table */ + for (taglist_t::const_iterator it = list->begin(); it != list->end(); ++it) { + lua_pushstring(L, it->key.c_str()); + lua_pushstring(L, it->value.c_str()); + lua_rawset(L, -3); + } + lua_rawset(L, -3); + } + + lua_newtable(L); /* member roles table */ + + for (size_t i = 0; i < member_roles.size(); i++) { + lua_pushnumber(L, i + 1); + lua_pushstring(L, member_roles[i]->c_str()); + lua_rawset(L, -3); + } + + lua_pushnumber(L, member_roles.size()); + + if (lua_pcall(L,4,6,0)) { + fprintf(stderr, "Failed to execute lua function for relation tag processing: %s\n", lua_tostring(L, -1)); + /* lua function failed */ + return 1; + } + + *roads = lua_tointeger(L, -1); + lua_pop(L,1); + *make_polygon = lua_tointeger(L, -1); + lua_pop(L,1); + *make_boundary = lua_tointeger(L,-1); + lua_pop(L,1); + + lua_pushnil(L); + for (size_t i = 0; i < member_tags.size(); i++) { + if (lua_next(L,-2)) { + member_superseeded[i] = lua_tointeger(L,-1); + lua_pop(L,1); + } else { + fprintf(stderr, "Failed to read member_superseeded from lua function\n"); + } + } + lua_pop(L,2); + + lua_pushnil(L); + while (lua_next(L,-2) != 0) { + const char *key = lua_tostring(L,-2); + const char *value = lua_tostring(L,-1); + out_tags.push_back(tag(key, value)); + lua_pop(L,1); + } + lua_pop(L,1); + + int filter = lua_tointeger(L, -1); + + lua_pop(L,1); + + return filter; +} + +void tagtransform::check_lua_function_exists(const std::string &func_name) +{ + lua_getglobal(L, func_name.c_str()); + if (!lua_isfunction (L, -1)) { + throw std::runtime_error((boost::format("Tag transform style does not contain a function %1%") + % func_name).str()); + } + lua_pop(L,1); +} +#endif + +tagtransform::tagtransform(const options_t *options_) + : options(options_), transform_method(options_->tag_transform_script) +#ifdef HAVE_LUA + , L(NULL) + , m_node_func( options->tag_transform_node_func. get_value_or("filter_tags_node")) + , m_way_func( options->tag_transform_way_func. get_value_or("filter_tags_way")) + , m_rel_func( options->tag_transform_rel_func. get_value_or("filter_basic_tags_rel")) + , m_rel_mem_func(options->tag_transform_rel_mem_func.get_value_or("filter_tags_relation_member")) +#endif /* HAVE_LUA */ +{ + if (transform_method) { + fprintf(stderr, "Using lua based tag processing pipeline with script %s\n", options->tag_transform_script->c_str()); +#ifdef HAVE_LUA + L = luaL_newstate(); + luaL_openlibs(L); + luaL_dofile(L, options->tag_transform_script->c_str()); + + check_lua_function_exists(m_node_func); + check_lua_function_exists(m_way_func); + check_lua_function_exists(m_rel_func); + check_lua_function_exists(m_rel_mem_func); +#else + throw std::runtime_error("Error: Could not init lua tag transform, as lua support was not compiled into this version"); +#endif + } else { + fprintf(stderr, "Using built-in tag processing pipeline\n"); + } +} + +tagtransform::~tagtransform() { +#ifdef HAVE_LUA + if (transform_method) + lua_close(L); +#endif +} + +unsigned int tagtransform::filter_node_tags(const taglist_t &tags, const export_list &exlist, + taglist_t &out_tags, bool strict) +{ + if (transform_method) { + return lua_filter_basic_tags(OSMTYPE_NODE, tags, 0, 0, out_tags); + } else { + return c_filter_basic_tags(OSMTYPE_NODE, tags, 0, 0, exlist, out_tags, strict); + } +} + +/* + * This function gets called twice during initial import per way. Once from add_way and once from out_way + */ +unsigned tagtransform::filter_way_tags(const taglist_t &tags, int *polygon, int *roads, + const export_list &exlist, taglist_t &out_tags, bool strict) +{ + if (transform_method) { + return lua_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads, out_tags); + } else { + return c_filter_basic_tags(OSMTYPE_WAY, tags, polygon, roads, exlist, out_tags, strict); + } +} + +unsigned tagtransform::filter_rel_tags(const taglist_t &tags, const export_list &exlist, + taglist_t &out_tags, bool strict) +{ + if (transform_method) { + return lua_filter_basic_tags(OSMTYPE_RELATION, tags, 0, 0, out_tags); + } else { + return c_filter_basic_tags(OSMTYPE_RELATION, tags, 0, 0, exlist, out_tags, strict); + } +} + +unsigned tagtransform::filter_rel_member_tags(const taglist_t &rel_tags, + const multitaglist_t &member_tags, const rolelist_t &member_roles, + int *member_superseeded, int *make_boundary, int *make_polygon, int *roads, + const export_list &exlist, taglist_t &out_tags, bool allow_typeless) +{ + if (transform_method) { +#ifdef HAVE_LUA + return lua_filter_rel_member_tags(rel_tags, member_tags, member_roles, member_superseeded, make_boundary, make_polygon, roads, out_tags); +#else + return 1; +#endif + } else { + return c_filter_rel_member_tags(rel_tags, member_tags, member_roles, member_superseeded, make_boundary, make_polygon, roads, exlist, out_tags, allow_typeless); + } +} + +unsigned tagtransform::lua_filter_basic_tags(OsmType type, const taglist_t &tags, + int *polygon, int *roads, taglist_t &out_tags) +{ +#ifdef HAVE_LUA + switch (type) { + case OSMTYPE_NODE: { + lua_getglobal(L, m_node_func.c_str()); + break; + } + case OSMTYPE_WAY: { + lua_getglobal(L, m_way_func.c_str()); + break; + } + case OSMTYPE_RELATION: { + lua_getglobal(L, m_rel_func.c_str()); + break; + } + } + + lua_newtable(L); /* key value table */ + + for (taglist_t::const_iterator it = tags.begin(); it != tags.end(); ++it) { + lua_pushstring(L, it->key.c_str()); + lua_pushstring(L, it->value.c_str()); + lua_rawset(L, -3); + } + + lua_pushinteger(L, tags.size()); + + if (lua_pcall(L,2,type == OSMTYPE_WAY ? 4 : 2,0)) { + fprintf(stderr, "Failed to execute lua function for basic tag processing: %s\n", lua_tostring(L, -1)); + /* lua function failed */ + return 1; + } + + if (type == OSMTYPE_WAY) { + assert(roads); + *roads = lua_tointeger(L, -1); + lua_pop(L,1); + assert(polygon); + *polygon = lua_tointeger(L, -1); + lua_pop(L,1); + } + + lua_pushnil(L); + while (lua_next(L,-2) != 0) { + const char *key = lua_tostring(L,-2); + const char *value = lua_tostring(L,-1); + out_tags.push_back(tag(key, value)); + lua_pop(L,1); + } + + int filter = lua_tointeger(L, -2); + + lua_pop(L,2); + + return filter; +#else + return 1; +#endif +} + +/* Go through the given tags and determine the union of flags. Also remove + * any tags from the list that we don't know about */ +unsigned int tagtransform::c_filter_basic_tags(OsmType type, const taglist_t &tags, int *polygon, + int *roads, const export_list &exlist, + taglist_t &out_tags, bool strict) +{ + //assume we dont like this set of tags + int filter = 1; + + int flags = 0; + int add_area_tag = 0; + + OsmType export_type; + if (type == OSMTYPE_RELATION) { + export_type = OSMTYPE_WAY; + } else { + export_type = type; + } + const std::vector &infos = exlist.get(export_type); + + /* We used to only go far enough to determine if it's a polygon or not, + but now we go through and filter stuff we don't need + pop each tag off and keep it in the temp list if we like it */ + for (taglist_t::const_iterator item = tags.begin(); item != tags.end(); ++item) { + //if we want to do more than the export list says + if(!strict) { + if (type == OSMTYPE_RELATION && "type" == item->key) { + out_tags.push_back(*item); + filter = 0; + continue; + } + /* Allow named islands to appear as polygons */ + if ("natural" == item->key && "coastline" == item->value) { + add_area_tag = 1; + + /* Discard natural=coastline tags (we render these from a shapefile instead) */ + if (!options->keep_coastlines) { + continue; + } + } + } + + //go through the actual tags found on the item and keep the ones in the export list + size_t i = 0; + for (; i < infos.size(); i++) { + const taginfo &info = infos[i]; + if (wildMatch(info.name.c_str(), item->key.c_str())) { + if (info.flags & FLAG_DELETE) { + break; + } + + filter = 0; + flags |= info.flags; + + out_tags.push_back(*item); + break; + } + } + + // if we didn't find any tags that we wanted to export + // and we aren't strictly adhering to the list + if (i == infos.size() && !strict) { + if (options->hstore_mode != HSTORE_NONE) { + /* with hstore, copy all tags... */ + out_tags.push_back(*item); + /* ... but if hstore_match_only is set then don't take this + as a reason for keeping the object */ + if (!options->hstore_match_only && "osm_uid" != item->key + && "osm_user" != item->key + && "osm_timestamp" != item->key + && "osm_version" != item->key + && "osm_changeset" != item->key) + filter = 0; + } else if (options->hstore_columns.size() > 0) { + /* does this column match any of the hstore column prefixes? */ + size_t j = 0; + for(; j < options->hstore_columns.size(); ++j) { + size_t pos = item->key.find(options->hstore_columns[j]); + if (pos == 0) { + out_tags.push_back(*item); + /* ... but if hstore_match_only is set then don't take this + as a reason for keeping the object */ + if (!options->hstore_match_only + && "osm_uid" != item->key + && "osm_user" != item->key + && "osm_timestamp" != item->key + && "osm_version" != item->key + && "osm_changeset" != item->key) + filter = 0; + break; + } + } + } + } + } + + if (polygon) { + if (add_area_tag) { + /* If we need to force this as a polygon, append an area tag */ + out_tags.push_dedupe(tag("area", "yes")); + *polygon = 1; + } else { + *polygon = tags.get_bool("area", flags & FLAG_POLYGON); + } + } + + if (roads && !filter && (type == OSMTYPE_WAY)) { + add_z_order(out_tags, roads); + } + + return filter; +} diff -Nru osm2pgsql-0.82.0/tagtransform.hpp osm2pgsql-0.88.1/tagtransform.hpp --- osm2pgsql-0.82.0/tagtransform.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tagtransform.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,59 @@ + +#ifndef TAGTRANSFORM_H +#define TAGTRANSFORM_H + +#include "config.h" +#include "osmtypes.hpp" + +#include + +struct options_t; +struct export_list; + +#ifdef HAVE_LUA +extern "C" { + #include +} +#endif + + + +class tagtransform { +public: + tagtransform(const options_t *options_); + ~tagtransform(); + + unsigned filter_node_tags(const taglist_t &tags, const export_list &exlist, + taglist_t &out_tags, bool strict = false); + unsigned filter_way_tags(const taglist_t &tags, int *polygon, int *roads, + const export_list &exlist, taglist_t &out_tags, bool strict = false); + unsigned filter_rel_tags(const taglist_t &tags, const export_list &exlist, + taglist_t &out_tags, bool strict = false); + unsigned filter_rel_member_tags(const taglist_t &rel_tags, + const multitaglist_t &member_tags, const rolelist_t &member_roles, + int *member_superseeded, int *make_boundary, int *make_polygon, int *roads, + const export_list &exlist, taglist_t &out_tags, bool allow_typeless = false); + +private: + unsigned lua_filter_basic_tags(OsmType type, const taglist_t &tags, + int *polygon, int *roads, taglist_t &out_tags); + unsigned c_filter_basic_tags(OsmType type, const taglist_t &tags, int *polygon, + int *roads, const export_list &exlist, + taglist_t &out_tags, bool strict); + unsigned int lua_filter_rel_member_tags(const taglist_t &rel_tags, + const multitaglist_t &member_tags, const rolelist_t &member_roles, + int *member_superseeded, int *make_boundary, int *make_polygon, int *roads, + taglist_t &out_tags); + void check_lua_function_exists(const std::string &func_name); + + + const options_t* options; + const bool transform_method; +#ifdef HAVE_LUA + lua_State *L; + const std::string m_node_func, m_way_func, m_rel_func, m_rel_mem_func; +#endif + +}; + +#endif //TAGTRANSFORM_H Binary files /tmp/UfI5DlM_YY/osm2pgsql-0.82.0/tests/000466354.osc.gz and /tmp/gTAyTkLKeZ/osm2pgsql-0.88.1/tests/000466354.osc.gz differ diff -Nru osm2pgsql-0.82.0/tests/common-pg.cpp osm2pgsql-0.88.1/tests/common-pg.cpp --- osm2pgsql-0.82.0/tests/common-pg.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/common-pg.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,176 @@ +#include "common-pg.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#include +#include +#define getpid _getpid +#define sleep Sleep +#endif + +namespace fs = boost::filesystem; + +namespace pg { + +conn_ptr conn::connect(const std::string &conninfo) { + return boost::shared_ptr(new conn(conninfo)); +} + +result_ptr conn::exec(const std::string &query) { + return boost::make_shared(shared_from_this(), query); +} + +result_ptr conn::exec(const boost::format &fmt) { + return exec(fmt.str()); +} + +PGconn *conn::get() { return m_conn; } + +conn::~conn() { + if (m_conn != NULL) { + PQfinish(m_conn); + m_conn = NULL; + } +} + +conn::conn(const std::string &conninfo) + : m_conn(PQconnectdb(conninfo.c_str())) { + if (PQstatus(m_conn) != CONNECTION_OK) { + std::ostringstream out; + out << "Could not connect to database \"" << conninfo + << "\" because: " << PQerrorMessage(m_conn); + PQfinish(m_conn); + m_conn = NULL; + throw std::runtime_error(out.str()); + } +} + +result::result(conn_ptr conn, const std::string &query) + : m_conn(conn), m_result(PQexec(conn->get(), query.c_str())) { + if (m_result == NULL) { + throw std::runtime_error((boost::format("Unable to run query \"%1%\": NULL result") + % query).str()); + } +} + +PGresult *result::get() { return m_result; } + +result::~result() { + if (m_result != NULL) { + PQclear(m_result); + m_result = NULL; + } +} + +tempdb::tempdb() + : m_conn(conn::connect("dbname=postgres")) { + result_ptr res = NULL; + m_db_name = (boost::format("osm2pgsql-test-%1%-%2%") % getpid() % time(NULL)).str(); + m_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % m_db_name); + //tests can be run concurrently which means that this query can collide with other similar ones + //so we implement a simple retry here to get around the case that they do collide if we dont + //we often fail due to both trying to access template1 at the same time + size_t retries = 0; + ExecStatusType status = PGRES_FATAL_ERROR; + while(status != PGRES_COMMAND_OK && retries++ < 20) + { + sleep(1); + res = m_conn->exec(boost::format("CREATE DATABASE \"%1%\" WITH ENCODING 'UTF8'") % m_db_name); + status = PQresultStatus(res->get()); + } + if (PQresultStatus(res->get()) != PGRES_COMMAND_OK) { + throw std::runtime_error((boost::format("Could not create a database: %1%") + % PQresultErrorMessage(res->get())).str()); + } + + m_conninfo = (boost::format("dbname=%1%") % m_db_name).str(); + conn_ptr db = conn::connect(m_conninfo); + + setup_extension(db, "postgis", "postgis-1.5/postgis.sql", "postgis-1.5/spatial_ref_sys.sql", NULL); + setup_extension(db, "hstore", NULL); +} + +void tempdb::check_tblspc() { + result_ptr res = m_conn->exec("SELECT spcname FROM pg_tablespace WHERE " + "spcname = 'tablespacetest'"); + if ((PQresultStatus(res->get()) != PGRES_TUPLES_OK) || + (PQntuples(res->get()) != 1)) { + std::ostringstream out; + out << "The test needs a temporary tablespace to run in, but it does not " + << "exist. Please create the temporary tablespace. On Linux, you can " + << "do this by running:\n" + << " sudo mkdir -p /tmp/psql-tablespace\n" + << " sudo /bin/chown postgres.postgres /tmp/psql-tablespace\n" + << " psql -c \"CREATE TABLESPACE tablespacetest LOCATION " + << "'/tmp/psql-tablespace'\" postgres\n"; + throw std::runtime_error(out.str()); + } + +} + +tempdb::~tempdb() { + if (m_conn) { + m_conn->exec(boost::format("DROP DATABASE IF EXISTS \"%1%\"") % m_db_name); + } +} + +const std::string &tempdb::conninfo() const { + return m_conninfo; +} + +void tempdb::setup_extension(conn_ptr db, const std::string &extension, ...) { + // first, try the new way of setting up extensions + result_ptr res = db->exec(boost::format("CREATE EXTENSION %1%") % extension); + if (PQresultStatus(res->get()) != PGRES_COMMAND_OK) { + // if that fails, then fall back to trying to find the files on + // the filesystem to load to create the extension. + res = db->exec("select regexp_replace(split_part(version(),' ',2),'\\.[0-9]*$','');"); + + if ((PQresultStatus(res->get()) != PGRES_TUPLES_OK) || + (PQntuples(res->get()) != 1)) { + throw std::runtime_error("Unable to determine PostgreSQL version."); + } + std::string pg_version(PQgetvalue(res->get(), 0, 0)); + + // Guess the directory from the postgres version. + // TODO: make the contribdir configurable. Probably + // only works on Debian-based distributions at the moment. + fs::path contribdir = fs::path("/usr/share/postgresql/") / pg_version / fs::path("contrib"); + va_list ap; + va_start(ap, extension); + const char *str = NULL; + while ((str = va_arg(ap, const char *)) != NULL) { + fs::path sql_file = contribdir / fs::path(str); + if (fs::exists(sql_file) && fs::is_regular_file(sql_file)) { + size_t size = fs::file_size(sql_file); + std::string sql(size + 1, '\0'); + fs::ifstream in(sql_file); + in.read(&sql[0], size); + if (in.fail() || in.bad()) { + throw std::runtime_error((boost::format("Unable to read file %1% while trying to " + "load extension \"%2%\".") + % sql_file % extension).str()); + } + res = db->exec(sql); + if (PQresultStatus(res->get()) != PGRES_COMMAND_OK) { + throw std::runtime_error((boost::format("Could not load extension \"%1%\": %2%") + % extension + % PQresultErrorMessage(res->get())).str()); + } + } + } + va_end(ap); + } +} +} // namespace pg diff -Nru osm2pgsql-0.82.0/tests/common-pg.hpp osm2pgsql-0.88.1/tests/common-pg.hpp --- osm2pgsql-0.82.0/tests/common-pg.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/common-pg.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,71 @@ +#ifndef COMMON_PG_HPP +#define COMMON_PG_HPP + +#include +#include +#include +#include +#include + +#include + +/* Some RAII objects to make writing stuff that needs a temporary database + * easier, and to keep track of and free connections and results objects. + */ +namespace pg { + +struct conn; +struct result; + +typedef boost::shared_ptr conn_ptr; +typedef boost::shared_ptr result_ptr; + +struct conn + : public boost::noncopyable, + public boost::enable_shared_from_this { + + static conn_ptr connect(const std::string &conninfo); + result_ptr exec(const std::string &query); + result_ptr exec(const boost::format &fmt); + PGconn *get(); + + ~conn(); + +private: + conn(const std::string &conninfo); + + PGconn *m_conn; +}; + +struct result + : public boost::noncopyable { + result(conn_ptr conn, const std::string &query); + PGresult *get(); + ~result(); + +private: + conn_ptr m_conn; + PGresult *m_result; +}; + +struct tempdb + : public boost::noncopyable { + + tempdb(); + ~tempdb(); + + const std::string &conninfo() const; + + void check_tblspc(); + +private: + void setup_extension(conn_ptr db, const std::string &extension, ...); + + conn_ptr m_conn; + std::string m_db_name; + std::string m_conninfo; +}; + +} // namespace pg + +#endif /* COMMON_PG_HPP */ Binary files /tmp/UfI5DlM_YY/osm2pgsql-0.82.0/tests/liechtenstein-2013-08-03.osm.bz2 and /tmp/gTAyTkLKeZ/osm2pgsql-0.88.1/tests/liechtenstein-2013-08-03.osm.bz2 differ Binary files /tmp/UfI5DlM_YY/osm2pgsql-0.82.0/tests/liechtenstein-2013-08-03.osm.pbf and /tmp/gTAyTkLKeZ/osm2pgsql-0.88.1/tests/liechtenstein-2013-08-03.osm.pbf differ diff -Nru osm2pgsql-0.82.0/tests/middle-tests.cpp osm2pgsql-0.88.1/tests/middle-tests.cpp --- osm2pgsql-0.82.0/tests/middle-tests.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/middle-tests.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,250 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "tests/middle-tests.hpp" + +#define BLOCK_SHIFT 10 +#define PER_BLOCK (((osmid_t)1) << BLOCK_SHIFT) + +struct expected_node { + osmid_t id; + double lon; + double lat; + + expected_node() : id(0), lon(NAN), lat(NAN) {} + + expected_node(osmid_t id, double x, double y) : id(id), lon(x), lat(y) {} +}; + +typedef std::vector expected_nodelist_t; + +#define ALLOWED_ERROR 10e-9 +bool node_okay(osmNode node, expected_node expected) { + if ((node.lat > expected.lat + ALLOWED_ERROR) || (node.lat < expected.lat - ALLOWED_ERROR)) { + std::cerr << "ERROR: Node should have lat=" << expected.lat << ", but got back " + << node.lat << " from middle.\n"; + return false; + } + if ((node.lon > expected.lon + ALLOWED_ERROR) || (node.lon < expected.lon - ALLOWED_ERROR)) { + std::cerr << "ERROR: Node should have lon=" << expected.lon << ", but got back " + << node.lon << " from middle.\n"; + return false; + } + return true; +} + +int test_node_set(middle_t *mid) +{ + idlist_t ids; + expected_node expected(1234, 12.3456789, 98.7654321); + taglist_t tags; + nodelist_t nodes; + + // set the node + if (mid->nodes_set(expected.id, expected.lat, expected.lon, tags) != 0) { std::cerr << "ERROR: Unable to set node.\n"; return 1; } + + // get it back + ids.push_back(expected.id); + if (mid->nodes_get_list(nodes, ids) != ids.size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; } + if (nodes.size() != ids.size()) { std::cerr << "ERROR: Mismatch in returned node list size.\n"; return 1; } + + // check that it's the same + if (!node_okay(nodes[0], expected)) { + return 1; + } + + return 0; +} + +inline double test_lat(osmid_t id) { + return 1 + 1e-5 * id; +} + +int test_nodes_comprehensive_set(middle_t *mid) +{ + taglist_t tags; + + expected_nodelist_t expected_nodes; + expected_nodes.reserve(PER_BLOCK*8+1); + + // 2 dense blocks, the second partially filled at the star + for (osmid_t id = 0; id < (PER_BLOCK+(PER_BLOCK >> 1) + 1); ++id) + { + expected_nodes.emplace_back(id, test_lat(id), 0.0); + } + + // 1 dense block, 75% filled + for (osmid_t id = PER_BLOCK*2; id < PER_BLOCK*3; ++id) + { + if ((id % 4 == 0) || (id % 4 == 1) || (id % 4 == 2)) + expected_nodes.emplace_back(id, test_lat(id), 0.0); + } + + // 1 dense block, sparsly filled + for (osmid_t id = PER_BLOCK*3; id < PER_BLOCK*4; ++id) + { + if (id % 4 == 0) + expected_nodes.emplace_back(id, test_lat(id), 0.0); + } + + // A lone sparse node + expected_nodes.emplace_back(PER_BLOCK*5, test_lat(PER_BLOCK*5), 0.0); + + // A dense block of alternating positions of zero/non-zero + for (osmid_t id = PER_BLOCK*6; id < PER_BLOCK*7; ++id) + { + if (id % 2 == 0) + expected_nodes.emplace_back(id, 0.0, 0.0); + else + expected_nodes.emplace_back(id, test_lat(id), 0.0); + } + expected_nodes.emplace_back(PER_BLOCK*8, 0.0, 0.0); + expected_nodes.emplace_back(PER_BLOCK*8+1, 0.0, 0.0); + + // Load up the nodes into the middle + idlist_t ids; + ids.reserve(expected_nodes.size()); + + for (expected_nodelist_t::iterator node = expected_nodes.begin(); node != expected_nodes.end(); ++node) + { + if (mid->nodes_set(node->id, node->lat, node->lon, tags) != 0) + { + std::cerr << "ERROR: Unable to set node " << node->id << "with lat=" + << node->lat << " lon=" << node->lon << std::endl; + return 1; + } + ids.push_back(node->id); + } + + nodelist_t nodes; + if (mid->nodes_get_list(nodes, ids) != ids.size()) { std::cerr << "ERROR: Unable to get node list.\n"; return 1; } + + if (nodes.size() != ids.size()) { std::cerr << "ERROR: Mismatch in returned node list size.\n"; return 1; } + + for (size_t i = 0; i < nodes.size(); ++i) + { + if (!node_okay(nodes[i], expected_nodes[i])) { + return 1; + } + } + return 0; +} + +struct test_pending_processor : public middle_t::pending_processor { + test_pending_processor(): pending_ways(), pending_rels() {} + virtual ~test_pending_processor() {} + virtual void enqueue_ways(osmid_t id) { + pending_ways.push_back(id); + } + virtual void process_ways() { + pending_ways.clear(); + } + virtual void enqueue_relations(osmid_t id) { + pending_rels.push_back(id); + } + virtual void process_relations() { + pending_rels.clear(); + } + virtual int thread_count() { + return 0; + } + virtual int size() { + return pending_ways.size() + pending_rels.size(); + } + std::list pending_ways; + std::list pending_rels; +}; + +int test_way_set(middle_t *mid) +{ + osmid_t way_id = 1; + double lat = 12.3456789; + double lon = 98.7654321; + taglist_t tags; + struct osmNode *node_ptr = NULL; + int status = 0; + idlist_t nds; + for (osmid_t i = 1; i <= 10; ++i) + nds.push_back(i); + + // set the nodes + for (size_t i = 0; i < nds.size(); ++i) { + status = mid->nodes_set(nds[i], lat, lon, tags); + if (status != 0) { std::cerr << "ERROR: Unable to set node " << nds[i] << ".\n"; return 1; } + } + + // set the way + status = mid->ways_set(way_id, nds, tags); + if (status != 0) { std::cerr << "ERROR: Unable to set way.\n"; return 1; } + + // commit the setup data + mid->commit(); + + // get it back + idlist_t ways, xways; + ways.push_back(way_id); + std::vector xtags; + multinodelist_t xnodes; + int way_count = mid->ways_get_list(ways, xways, xtags, xnodes); + if (way_count != 1) { std::cerr << "ERROR: Unable to get way list.\n"; return 1; } + + // check that it's the same + if (xnodes[0].size() != nds.size()) { + std::cerr << "ERROR: Way should have " << nds.size() << " nodes, but got back " + << xnodes[0].size() << " from middle.\n"; + return 1; + } + if (xways[0] != way_id) { + std::cerr << "ERROR: Way should have id=" << way_id << ", but got back " + << xways[0] << " from middle.\n"; + return 1; + } + for (size_t i = 0; i < nds.size(); ++i) { + if (xnodes[0][i].lon != lon) { + std::cerr << "ERROR: Way node should have lon=" << lon << ", but got back " + << node_ptr[i].lon << " from middle.\n"; + return 1; + } + if (xnodes[0][i].lat != lat) { + std::cerr << "ERROR: Way node should have lat=" << lat << ", but got back " + << node_ptr[i].lat << " from middle.\n"; + return 1; + } + } + + // the way we just inserted should not be pending + test_pending_processor tpp; + mid->iterate_ways(tpp); + if (mid->pending_count() != 0) { + std::cerr << "ERROR: Was expecting no pending ways, but got " + << mid->pending_count() << " from middle.\n"; + return 1; + } + + // some middles don't support changing the nodes - they + // don't have diff update ability. here, we will just + // skip the test for that. + if (dynamic_cast(mid)) { + slim_middle_t *slim = dynamic_cast(mid); + + // finally, try touching a node on a non-pending way. that should + // make it become pending. we just checked that the way is not + // pending, so any change must be due to the node changing. + status = slim->node_changed(nds[0]); + if (status != 0) { std::cerr << "ERROR: Unable to reset node.\n"; return 1; } + slim->iterate_ways(tpp); + if (slim->pending_count() != 1) { + std::cerr << "ERROR: Was expecting a single pending way from node update, but got " + << slim->pending_count() << " from middle.\n"; + return 1; + } + } + + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/middle-tests.hpp osm2pgsql-0.88.1/tests/middle-tests.hpp --- osm2pgsql-0.82.0/tests/middle-tests.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/middle-tests.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,16 @@ +#ifndef TESTS_MIDDLE_TEST_HPP +#define TESTS_MIDDLE_TEST_HPP + +#include "middle.hpp" + +// tests that a single node can be set and retrieved. returns 0 on success. +int test_node_set(middle_t *mid); + +// tests various combinations of nodes being set and retrieved to trigger different cache strategies. returns 0 on success. +int test_nodes_comprehensive_set(middle_t *mid); + +// tests that a single way and supporting nodes can be set and retrieved. +// returns 0 on success. +int test_way_set(middle_t *mid); + +#endif /* TESTS_MIDDLE_TEST_HPP */ diff -Nru osm2pgsql-0.82.0/tests/regression-test.py osm2pgsql-0.88.1/tests/regression-test.py --- osm2pgsql-0.82.0/tests/regression-test.py 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/regression-test.py 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,644 @@ +#!/usr/bin/env python + +import unittest +import psycopg2 +import os +import fnmatch +import subprocess + +full_import_file="tests/liechtenstein-2013-08-03.osm.pbf" +multipoly_import_file="tests/test_multipolygon.osm" #This file contains a number of different multi-polygon test cases +diff_import_file="tests/000466354.osc.gz" +diff_multipoly_import_file="tests/test_multipolygon_diff.osc" #This file contains a number of different multi-polygon diff processing test cases + +created_tablespace = 0 + +#**************************************************************** +#**************************************************************** +sql_test_statements=[ + ( 0, 'Basic point count', 'SELECT count(*) FROM planet_osm_point;', 1342 ), + ( 1, 'Basic line count', 'SELECT count(*) FROM planet_osm_line;', 3300 ), + ( 2, 'Basic road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ), + ( 3, 'Basic polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ), + ( 4, 'Basic latlon line count', 'SELECT count(*) FROM planet_osm_line;', 3298 ), + ( 5, 'Basic latlon road count', 'SELECT count(*) FROM planet_osm_roads;', 374 ), + ( 6, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1457 ), + ( 7, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3344 ), + ( 8, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ), + ( 9, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ), + ( 10, 'Absence of nodes table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_nodes\'', 0), + ( 11, 'Absence of way table', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_ways\'', 0), + ( 12, 'Absence of rel line', 'SELECT count(*) FROM pg_tables WHERE tablename = \'planet_osm_rels\'', 0), + ( 13, 'Basic polygon area', 'SELECT round(sum(cast(ST_Area(way) as numeric)),0) FROM planet_osm_polygon;', 1223800814), + ( 14, 'Gazetteer place count', 'SELECT count(*) FROM place', 2837), + ( 15, 'Gazetteer place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 759), + ( 16, 'Gazetteer place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2059), + ( 17, 'Gazetteer place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19), + ( 18, 'Gazetteer post-diff place count', 'SELECT count(*) FROM place', 2878), + ( 19, 'Gazetteer post-diff place node count', 'SELECT count(*) FROM place WHERE osm_type = \'N\'', 764), + ( 20, 'Gazetteer post-diff place way count', 'SELECT count(*) FROM place WHERE osm_type = \'W\'', 2095), + ( 21, 'Gazetteer post-diff place rel count', 'SELECT count(*) FROM place WHERE osm_type = \'R\'', 19), + ( 22, 'Gazetteer housenumber count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199), + ( 23, 'Gazetteer post-diff housenumber count count', 'SELECT count(*) FROM place WHERE housenumber is not null', 199), + ( 24, 'Gazetteer isin count', 'SELECT count(*) FROM place WHERE isin is not null', 239), + ( 25, 'Gazetteer post-diff isin count count', 'SELECT count(*) FROM place WHERE isin is not null', 239), + ( 26, 'Multipolygon basic case (Tags from outer way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -15 and landuse = \'residential\' and name = \'Name_way\'', 12894), + ( 27, 'Multipolygon basic case (Tags from relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -1 and landuse = \'residential\' and name = \'Name_rel\'', 12895), + ( 28, 'Multipolygon named inner - outer (Tags from way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -16 and landuse = \'residential\' and name = \'Name_way2\'', 12895), + ( 29, 'Multipolygon named inner - inner way', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 4 and landuse = \'farmland\' and name = \'Name_way3\'', 3144), + ( 30, 'Multipolygon named inner - outer (Tags from relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -8 and landuse = \'residential\' and name = \'Name_rel2\'', 12894), + ( 31, 'Multipolygon named inner - inner way', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 5 and landuse = \'farmland\' and name = \'Name_way4\'', 3144), + ( 32, 'Multipolygon named same inner - outer (Tags from way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -17 and landuse = \'residential\' and name = \'Name_way16\'', 12895), + ( 33, 'Multipolygon named same inner - inner way absent', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 15', 0), + ( 34, 'Multipolygon non-area inner - outer (Tags from relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -14 and landuse = \'residential\' and name = \'Name_way5\'', 12893), + ( 35, 'Multipolygon non-area inner - inner (Tags from way)', + 'SELECT round(ST_Length(way)) FROM planet_osm_line WHERE osm_id = 6 and highway = \'residential\' and name = \'Name_way6\'', 228), + ( 36, 'Multipolygon 2 holes (Tags from way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -18 and landuse = \'residential\' and name = \'Name_way7\'', 11823), + ( 37, 'Multipolygon 2 holes (Tags from way)', + 'SELECT ST_NumInteriorRing(way) FROM planet_osm_polygon WHERE osm_id = -18 and landuse = \'residential\' and name = \'Name_way7\'', 2), + ( 38, 'Multipolygon from multiple outer ways 0 holes (Tags from relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -11 and landuse = \'residential\' and name = \'Name_rel6\'', 11528), + ( 39, 'Multipolygon from multiple outer and multiple inner ways 2 holes (Tags from relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -3 and landuse = \'residential\' and name = \'Name_rel11\'', 9286), + ( 40, 'Multipolygon 2 holes (Tags from way)', + 'SELECT ST_NumInteriorRing(way) FROM planet_osm_polygon WHERE osm_id = -3 and landuse = \'residential\' and name = \'Name_rel11\'', 2), + ( 41, 'Multipolygon with touching inner ways 1 hole (Tags from way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -19 and landuse = \'residential\' and name = \'Name_way8\'', 12167), + ( 42, 'Multipolygon with touching inner ways 1 hole (Tags from way)', + 'SELECT ST_NumInteriorRing(way) FROM planet_osm_polygon WHERE osm_id = -19 and landuse = \'residential\' and name = \'Name_way8\'', 1), + ( 43, 'Multipolygon with 2 outer ways (Tags from relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -13 and landuse = \'farmland\' and name = \'Name_rel9\'', 17581), + ( 44, 'Multipolygon with 2 outer ways (Tags from relation)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -13 and landuse = \'farmland\' and name = \'Name_rel9\'', 2), + ( 45, 'Multipolygon with 2 outer ways (multigeometry)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -13 and landuse = \'farmland\' and name = \'Name_rel9\'', 1), + ( 46, 'Multipolygon with 2 outer ways (multigeometry)', + 'SELECT ST_NumGeometries(way) FROM planet_osm_polygon WHERE osm_id = -13 and landuse = \'farmland\' and name = \'Name_rel9\'', 2), + ( 47, 'Multipolygon nested outer ways. Both outer and inner ways are from multiple ways (Tags from relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -7 and landuse = \'farmland\' and name = \'Name_rel15\'', 16169), + ( 48, 'Multipolygon nested outer ways. Both outer and inner ways are from multiple ways (Tags from relation)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -7 and landuse = \'farmland\' and name = \'Name_rel15\'', 2), + ( 49, 'Multipolygon nested outer ways. Both outer and inner ways are from multiple ways (multigeometry)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -7 and landuse = \'farmland\' and name = \'Name_rel15\'', 1), + ( 50, 'Multipolygon nested outer ways. Both outer and inner ways are from multiple ways (multigeometry)', + 'SELECT ST_NumGeometries(way) FROM planet_osm_polygon WHERE osm_id = -7 and landuse = \'farmland\' and name = \'Name_rel15\'', 2), + ( 51, 'Basic hstore point count', 'SELECT count(*) FROM planet_osm_point;', 1360 ), + ( 52, 'Basic hstore line count', 'SELECT count(*) FROM planet_osm_line;', 3323 ), + ( 53, 'Basic hstore road count', 'SELECT count(*) FROM planet_osm_roads;', 375 ), + ( 54, 'Basic hstore polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4128 ), + ( 55, 'Basic post-diff point count', 'SELECT count(*) FROM planet_osm_point;', 1475 ), + ( 56, 'Basic post-diff line count', 'SELECT count(*) FROM planet_osm_line;', 3367 ), + ( 57, 'Basic post-diff road count', 'SELECT count(*) FROM planet_osm_roads;', 381 ), + ( 58, 'Basic post-diff polygon count', 'SELECT count(*) FROM planet_osm_polygon;', 4275 ), + ( 59, 'Extra hstore full tags point count', + 'SELECT count(*) FROM planet_osm_point WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 1360), + ( 60, 'Extra hstore full tags line count', + 'SELECT count(*) FROM planet_osm_line WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 3323), + ( 61, 'Extra hstore full tags polygon count', + 'SELECT count(*) FROM planet_osm_polygon WHERE tags ? \'osm_user\' and tags ? \'osm_version\' and tags ? \'osm_uid\' and tags ? \'osm_changeset\'', 4128), + ( 62, 'Multipolygon copying of tags from outer with extra tags on relation', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -22', 20879), + ( 63, 'Multipolygon copying of tags from outer with extra tags on relation (abscence of way)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 84', 0), + ( 64, 'Multipolygon non copying of tags from outer with polygon tags on relation', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -24 and "natural" = \'water\'', 18501), + ( 65, 'Multipolygon non copying of tags from outer with polygon tags on relation (presence of way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 83 and "landuse" = \'farmland\'', 24859), + ( 66, 'Multipolygon diff moved point of outer way case (Tags from outer way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -15 and landuse = \'residential\' and name = \'Name_way\'', 24750), + ( 67, 'Multipolygon diff moved point of inner way case (Tags from relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -1 and landuse = \'residential\' and name = \'Name_rel\'', 13949), + ( 68, 'Multipolygon point of inner way case (Tags from relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -25 and landuse = \'farmland\' and name = \'my name\'', 23886), + ( 69, 'Multipolygon point of inner way case (Tags from relation)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 90', 0), + ( 70, 'Multipolygon diff remove relation (tagged outer way gets re added)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 90 and landuse = \'farmland\'', 32626), + ( 71, 'Multipolygon diff remove relation', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -25', 0), + ( 72, 'Multipolygon tags on both inner and outer (presence of relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -34 and "natural" = \'water\'', 15246), + ( 73, 'Multipolygon tags on both inner and outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 113', 0), + ( 74, 'Multipolygon tags on both inner and outer (abscence of inner)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 118', 0), + ( 75, 'Multipolygon tags on both inner and outer diff change outer (presence of relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -34 and "landuse" = \'farmland\'', 15246), + ( 76, 'Multipolygon tags on both inner and outer diff change outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 113', 0), + ( 77, 'Multipolygon tags on both inner and outer diff change on outer (creation of inner)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 118 and "natural" = \'water\'', 1234), + ( 78, 'Multipolygon tags on outer (presence of relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -33 and "natural" = \'water\'', 15612), + ( 79, 'Multipolygon tags on outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 114', 0), + ( 80, 'Multipolygon tags on outer change of way tags (presence of relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -33 and "landuse" = \'cemetery\'', 15612), + ( 81, 'Multipolygon tags on outer (abscence of old relation)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -33 and "natural" = \'water\'', 0), + ( 82, 'Multipolygon tags on relation two outer (presence of relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -29 and "natural" = \'water\'', 68492), + ( 83, 'Multipolygon tags on relation two outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 109', 0), + ( 84, 'Multipolygon tags on relation two outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 104', 0), + ( 85, 'Multipolygon tags on relation two outer diff delete way (presence of relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -29 and "natural" = \'water\'', 29154), + ( 86, 'Multipolygon tags on relation two outer (presence of relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 28730), + ( 87, 'Multipolygon tags on relation two outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 107', 0), + ( 88, 'Multipolygon tags on relation two outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 102', 0), + ( 89, 'Multipolygon tags on relation two outer diff remove way from relation (presence of relation)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = -35 and "natural" = \'water\'', 15736), + ( 90, 'Multipolygon tags on relation two outer diff remove way from relation (presence of single way)', + 'SELECT round(ST_Area(way)) FROM planet_osm_polygon WHERE osm_id = 102 and "natural" = \'water\'', 12994), + ( 91, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_line;', 4269394), + ( 92, 'Basic line length', 'SELECT round(sum(ST_Length(way))) FROM planet_osm_roads;', 2032023), + ( 93, 'Basic number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4228), + ( 94, 'Basic number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2316), + ( 95, 'Basic number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11131), + ( 96, 'Basic number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9540), + ( 97, 'Diff import number of hstore points tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_point;', 4352), + ( 98, 'Diff import number of hstore roads tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_roads;', 2340), + ( 99, 'Diff import number of hstore lines tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_line;', 11254), + ( 100, 'Diff import number of hstore polygons tags', 'SELECT sum(array_length(akeys(tags),1)) FROM planet_osm_polygon;', 9834), + #**** Tests to check if inner polygon appears when outer tags change after initially identicall inner and outer way tags in a multi-polygon **** + #**** These tests are currently broken and noted in trac ticket #2853 **** + ( 101, 'Multipolygon identical tags on inner and outer (presence of relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -31 and "natural" = \'heath\'', 32702), + ( 102, 'Multipolygon identical tags on inner and outer (abscence of outer)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 120', 0), + ( 103, 'Multipolygon identical tags on inner and outer (abscence of inner)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 112', 0), + ( 104, 'Multipolygon identical tags on inner and outer (presence of relation), post diff', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -31 and "natural" = \'water\'', 32702), + ( 105, 'Multipolygon identical tags on inner and outer (presece of inner)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = 112 and "natural" = \'heath\'', 1234), + #**** Test to check that only polygon tags that are present on all outer ways get copied over to the multi-polygon relation **** + ( 106, 'Multipolygon copy outer tags (presence of relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -38 and "natural" = \'water\'', 29340), + ( 107, 'Multipolygon copy outer tags (absence of partial outer tags)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -38 and "natural" = \'water\' and "man_made" = \'pier\'', 0), + ( 108, 'Multipolygon copy outer tags (absence of multi-polygon tagged outer way)', + 'SELECT count(*) FROM planet_osm_line WHERE osm_id = 134 OR osm_id = 133', 0), + ( 109, 'Multipolygon copy outer tags (presence of additionally tagged outer way)', + 'SELECT round(sum(ST_length(way))) FROM planet_osm_line WHERE (osm_id = 136 OR osm_id = 132) AND "man_made" = \'pier\'', 407), + ( 110, 'Multipolygon copy outer tags (presence of relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -37 and "natural" = \'water\'', 29952), + ( 111, 'Multipolygon copy outer tags (absence of partial outer tags)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = -37 and "natural" = \'water\' and "man_made" = \'pier\'', 0), + ( 112, 'Multipolygon copy outer tags (absence of multi-polygon tagged outer way)', + 'SELECT count(*) FROM planet_osm_line WHERE osm_id = 128 OR osm_id = 125', 0), + ( 113, 'Multipolygon copy outer tags (presence of additionally tagged outer way)', + 'SELECT round(sum(ST_length(way))) FROM planet_osm_line WHERE (osm_id = 126 OR osm_id = 124) AND "man_made" = \'pier\'', 276), + ( 114, 'Multipolygon copy outer tags (absence of multi-polygon tagged inner way)', + 'SELECT count(*) FROM planet_osm_line WHERE osm_id = 123 OR osm_id = 121', 0), + ( 115, 'Multipolygon copy outer tags (presence of additionally tagged inner way)', + 'SELECT round(sum(ST_length(way))) FROM planet_osm_line WHERE (osm_id = 127 OR osm_id = 122) AND "man_made" = \'pier\'', 318), + #**** Test to check that if polygon tags are on both outer ways and relation, polygons don't get duplicated in the db **** + ( 116, 'Multipolygon tags on both outer and relation (presence of relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -39 and "landuse" = \'forest\'', 10379), + ( 117, 'Multipolygon tags on both outer and relation (absence of outer way)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 138', 0), + ( 118, 'Multipolygon tags on both outer and relation with additional tags on relation (presence of relation)', + 'SELECT round(sum(ST_Area(way))) FROM planet_osm_polygon WHERE osm_id = -40 and "landuse" = \'forest\'', 12397), + ( 119, 'Multipolygon tags on both outer and relation with additional tags on relation (absence of outer way)', + 'SELECT count(*) FROM planet_osm_polygon WHERE osm_id = 140', 0), + ] +#**************************************************************** +#**************************************************************** + + +class NonSlimRenderingTestSuite(unittest.TestSuite): + def __init__(self): + unittest.TestSuite.__init__(self,map(ThirdTestCase, + ("testOne", + "testTwo"))) + self.addTest(BasicNonSlimTestCase("basic case",[], [0,1,2,3,10,13, 91, 92])) + self.addTest(BasicNonSlimTestCase("slim --drop case",["--slim","--drop"], [0,1,2,3, 10, 11, 12, 13, 91, 92])) + self.addTest(BasicNonSlimTestCase("Hstore index drop", ["--slim", "--hstore", "--hstore-add-index", "--drop"], [51,52,53,54])) + self.addTest(BasicNonSlimTestCase("lat lon projection",["-l"], [0,4,5,3,10, 11, 12])) + #Failing test 3,13 due to difference in handling mixture of tags on ways and relations, where the correct behaviour is non obvious + #self.addTest(BasicNonSlimTestCase("--tag-transform-script", ["--tag-transform-script", "style.lua"], [0,1,2,3,10,13,91,92])) + self.addTest(BasicNonSlimTestCase("--tag-transform-script", ["--tag-transform-script", "style.lua"], [0,1,2,10,91,92])) + + +class SlimRenderingTestSuite(unittest.TestSuite): + def __init__(self): + unittest.TestSuite.__init__(self,map(ThirdTestCase, + ("testOne", + "testTwo"))) + self.addTest(BasicSlimTestCase("basic case", [], [0,1,2,3,13, 91, 92],[6,7,8,9])) + self.addTest(BasicSlimTestCase("Parallel processing", ["--number-processes", "8", "-C100"], [0,1,2,3,13,91,92],[6,7,8,9])) + self.addTest(BasicSlimTestCase("Parallel processing with non 100% node-cache", ["--number-processes", "8", "-C1", "--cache-strategy=dense"], [0,1,2,3,13,91,92],[6,7,8,9])) + self.addTest(BasicSlimTestCase("Parallel processing with disabled node-cache", ["-C0"], [0,1,2,3,13,91,92],[6,7,8,9])) + # Failes to do correct error checking. This needs fixing in osm2pgsql + # self.addTest(BasicSlimTestCase("Parallel processing with failing database conneciton (connection limit exceeded)", ["--number-processes", "32", "-C100"], [0,1,2,3],[6,7,8,9])) + # Counts are expected to be different in hstore, needs adjusted tests + self.addTest(BasicSlimTestCase("Hstore match only", ["-k", "--hstore-match-only"], [0,1,2,3],[6,7,8,9])) + self.addTest(BasicSlimTestCase("Hstore name column", ["-z", "name:"], [0,1,2,3],[6,7,8,9])) + self.addTest(BasicSlimTestCase("Hstore", ["-k"], [51,52,53,54],[55,56,57,58])) + self.addTest(BasicSlimTestCase("Hstore all", ["-j"], [51,52,53,54,93,94,95,96],[55,56,57,58, 97, 98, 99, 100])) + self.addTest(BasicSlimTestCase("Hstore index", ["--hstore", "--hstore-add-index"], [51,52,53,54],[55,56,57,58])) + #tests dont check for osm_timestamp which is currently missing in the pbf parser + self.addTest(BasicSlimTestCase("Extra tags hstore match only", ["-x", "-k", "--hstore-match-only"], [0,1,2,3],[6,7,8,9])) + self.addTest(BasicSlimTestCase("Extra tags hstore all", ["-j", "-x"], [51,52,53,54,59,60,61],[55,56,57,58])) + + self.addTest(BasicSlimTestCase("--tablespace-main-data", ["--tablespace-main-data", "tablespacetest"], [0,1,2,3,13,91,92],[6,7,8,9])) + self.addTest(BasicSlimTestCase("--tablespace-main-index", ["--tablespace-main-index", "tablespacetest"], [0,1,2,3,13,91,92],[6,7,8,9])) + self.addTest(BasicSlimTestCase("--tablespace-slim-data", ["--tablespace-slim-data", "tablespacetest"], [0,1,2,3,13,91,92],[6,7,8,9])) + self.addTest(BasicSlimTestCase("--tablespace-slim-index", ["--tablespace-slim-index", "tablespacetest"], [0,1,2,3,13,91,92],[6,7,8,9])) + #Failing test 3,13,9 due to difference in handling mixture of tags on ways and relations, where the correct behaviour is non obvious + #self.addTest(BasicNonSlimTestCase("--tag-transform-script", ["--tag-transform-script", "style.lua"], [0,1,2,3,10,13,91,92])) + self.addTest(BasicSlimTestCase("--tag-transform-script", ["--tag-transform-script", "style.lua"], [0,1,2,91,92],[6,7,8])) + + +class SlimGazetteerTestSuite(unittest.TestSuite): + def __init__(self): + unittest.TestSuite.__init__(self,map(ThirdTestCase, + ("testOne", + "testTwo"))) + self.addTest(BasicGazetteerTestCase("basic case", [], [14,15,16,17,22,24],[18,19,20,21,23,25])) + + +class MultiPolygonSlimRenderingTestSuite(unittest.TestSuite): + def __init__(self): + unittest.TestSuite.__init__(self,map(ThirdTestCase, + ("testOne", + "testTwo"))) + #Case 77 currently doesn't work + self.addTest(MultipolygonSlimTestCase("basic case", [], + [26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 68, 69, 72, 73, 74, 78, 79, 82, 83, 84, 86, 87, 88, + 106,107,108,109,110,111,112,113,114,115,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 66, 67, 70, 71, 75, 76, 79, 80, 81, 83, 84, 85, 87, 89, 90])) + self.addTest(MultipolygonSlimTestCase("multi geometry", ["-G"], + [26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 45, 46, 47, 49, 50, 62, 63, 64, 65, 68, 69, 72, 73, 74, 78, 79, 82, 83, 84, 86, 87, 88, + 106,107,108,109,110,111,112,113,114,115,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 45, 46, 47, 49, 50, 62, 63, 64, 65, 66, 67, 70, 71, 75, 76, 79, 80, 81, 83, 84, 85, 87, 89, 90])) + self.addTest(MultipolygonSlimTestCase("hstore case", ["-k"], + [26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,47,48,62,63,64,65,68,69, 72, 73, 74, 78, 79, 82, 83, 84, 86, 87, 88, + 106,107,108,109,110,111,112,113,114,115,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 66, 67, 70, 71, 75, 76, 79, 80, 81, 83, 84, 85, 87, 89, 90])) + self.addTest(MultipolygonSlimTestCase("hstore case", ["-k", "--hstore-match-only"], + [26,27,28,29,30,31,32,33,34,35,36,37,38, 39, 40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 68, 69, 72, 73, 74, 78, 79, 82, 83, 84, 86, 87, 88, + 106,107,108,109,110,111,112,113,114,115,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 66, 67, 70, 71, 75, 76, 79, 80, 81, 83, 84, 85, 87, 89, 90])) + self.addTest(MultipolygonSlimTestCase("Extra tags hstore match only", ["-x", "-k", "--hstore-match-only"], + [26,27,28,29,30,31,32,33,34,35,36,37,38, 39, 40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 68, 69, 72, 73, 74, 78, 79, 82, 83, 84, 86, 87, 88, + 106,107,108,109,110,111,112,113,114,115,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 66, 67, 70, 71, 75, 76, 79, 80, 81, 83, 84, 85, 87, 89, 90])) + self.addTest(MultipolygonSlimTestCase("Extra tags hstore match only", ["-x", "-j"], + [26,27,28,29,30,31,32,33,34,35,36,37,38, 39, 40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 68, 69, 72, 73, 74, 78, 79, 82, 83, 84, 86, 87, 88, + 106,107,108,109,110,111,112,113,114,115,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 44, 47, 48, 62, 63, 64, 65, 66, 67, 70, 71, 75, 76, 79, 80, 81, 83, 84, 85, 87, 89, 90])) + self.addTest(MultipolygonSlimTestCase("lua tagtransform case", ["--tag-transform-script", "style.lua"], + [26,27,28,29,30,31,32,33,34,35,36,37,38, 39, 40, 41, 42, 43, 44, 47, 48, 62, 64, 65,68,69, 72, 73, 74, 78, 79, 82, 83, 84, 86, 87, 88,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42, 43, 44, 47, 48, 62, 63,64, 65, 66, 67, 70, 71, 75, 76, 79, 80, 81, 83, 84, 85, 87, 89, 90])) + self.addTest(MultipolygonSlimTestCase("lua tagtransform case with hstore", ["--tag-transform-script", "style.lua", "-k"], + [26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,47,48,62,63,64,65,68,69,72,73,74,78,79,82,83,84,86,87,88,116,117,118,119], + [28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,47,48,62,63,64,65,66,67,70,71,75,76,79,80,81,83,84,85,87,89,90])) + + +class CompleteTestSuite(unittest.TestSuite): + def __init__(self): + unittest.TestSuite.__init__(self, map(ThirdTestCase, + ("testOne", + "testTwo"))) + self.addTest(NonSlimRenderingTestSuite()) + self.addTest(SlimRenderingTestSuite()) + self.addTest(MultiPolygonSlimRenderingTestSuite()) + self.addTest(SlimGazetteerTestSuite()) + +#**************************************************************** +class ThirdTestCase(unittest.TestCase): + def testOne(self): + assert 1 == 1 + def testTwo(self): + assert 2 == 2 + +#**************************************************************** + +class BaseTestCase(unittest.TestCase): + def dbConnect(self): + try: + self.conn=psycopg2.connect("dbname='osm2pgsql-test'") + self.conn.autocommit = True + self.cur = self.conn.cursor() + except Exception as e: + print "I am unable to connect to the database." + e + + def dbClose(self): + self.cur.close() + self.conn.close() + + def executeStatements(self, seq): + print "*********************************" + self.dbConnect() + try: + for i in seq: + self.assertEqual(sql_test_statements[i][0], i, "test case numbers don't match up: " + str(i) + " =/=" + str(sql_test_statements[i][0])) + try: + self.cur.execute(sql_test_statements[i][2]) + res = self.cur.fetchall() + except Exception as e: + self.assertEqual(0, 1, str(sql_test_statements[i][0]) + ": Failed to execute " + sql_test_statements[i][1] + + " (" + sql_test_statements[i][2] + ") {" + str(self.parameters) +"}") + if (res == None): + self.assertEqual(0, 1, str(sql_test_statements[i][0]) + ": Sql statement returned no results: " + + sql_test_statements[i][1] + " (" + sql_test_statements[i][2] + ") {" + str(self.parameters) +"}") + self.assertEqual(len(res), 1, str(sql_test_statements[i][0]) + ": Sql statement returned more than one result: " + + str(res) + " -- " + sql_test_statements[i][1] + " (" + sql_test_statements[i][2] + ") {" + str(self.parameters) +"}") + self.assertEqual( res[0][0], sql_test_statements[i][3], + str(sql_test_statements[i][0]) + ": Failed " + sql_test_statements[i][1] + ", expected " + str(sql_test_statements[i][3]) + " but was " + str(res[0][0]) + + " (" + sql_test_statements[i][2] + ") {" + str(self.parameters) +"}") + finally: + self.dbClose() + +#**************************************************************** + +class BaseNonSlimTestCase(BaseTestCase): + + def setUpGeneric(self, parameters, file): + proc = subprocess.Popen(["./osm2pgsql", "-Sdefault.style", "-dosm2pgsql-test", "-C100"] + parameters + [full_import_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (outp, outerr) = proc.communicate() + self.assertEqual (proc.returncode, 0, "Execution of osm2pgsql with options: '%s' failed:\n%s\n%s\n" % (str(parameters), outp, outerr)) + +class BaseSlimTestCase(BaseTestCase): + + def setUpGeneric(self, parameters, file): + proc = subprocess.Popen(["./osm2pgsql", "--slim", "-Sdefault.style", "-dosm2pgsql-test", "-C100"] + parameters + [file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (outp, outerr) = proc.communicate() + self.assertEqual (proc.returncode, 0, "Execution of osm2pgsql --slim with options: '%s' failed:\n%s\n%s\n" % (str(parameters), outp, outerr)) + + def updateGeneric(self, parameters, file): + proc = subprocess.Popen(["./osm2pgsql", "--slim", "--append", "-Sdefault.style", "-dosm2pgsql-test", "-C100"] + parameters + [file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (outp, outerr) = proc.communicate() + self.assertEqual (proc.returncode, 0, "Execution of osm2pgsql --slim --append with options: '%s' failed:\n%s\n%s\n" % (str(parameters), outp, outerr)) + +class BaseGazetteerTestCase(BaseTestCase): + + def setUpGeneric(self, parameters, file): + proc = subprocess.Popen(["./osm2pgsql", "--slim", "-Ogazetteer", "-Sdefault.style", "-dosm2pgsql-test"] + parameters + [file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (outp, outerr) = proc.communicate() + self.assertEqual (proc.returncode, 0, "Execution of osm2pgsql --slim gazetteer options: '%s' failed:\n%s\n%s\n" % (str(parameters), outp, outerr)) + + def updateGeneric(self, parameters, file): + proc = subprocess.Popen(["./osm2pgsql", "--slim", "-Ogazetteer", "--append", "-Sdefault.style", "-dosm2pgsql-test"] + parameters + [file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (outp, outerr) = proc.communicate() + self.assertEqual (proc.returncode, 0, "Execution of osm2pgsql --slim --append gazetteer options: '%s' failed:\n%s\n%s\n" % (str(parameters), outp, outerr)) + + +#**************************************************************** +class BasicNonSlimTestCase(BaseNonSlimTestCase): + + def __init__(self, name, parameters, initialStatements): + BaseNonSlimTestCase.__init__(self) + self.name = name + self.parameters = parameters + self.initialStatements = initialStatements + + def setUp(self): + self.setUpGeneric(self.parameters, full_import_file) + + def runTest(self): + print "****************************************" + print "Running initial import for " + self.name + self.executeStatements(self.initialStatements) + + +class BasicSlimTestCase(BaseSlimTestCase): + + def __init__(self, name, parameters, initialStatements, postDiffStatements): + BaseSlimTestCase.__init__(self) + self.name = name + self.parameters = parameters + self.initialStatements = initialStatements + self.postDiffStatements = postDiffStatements + + def setUp(self): + self.setUpGeneric(self.parameters, full_import_file) + + + def runTest(self): + print "****************************************" + print "Running initial import for " + self.name + self.executeStatements(self.initialStatements) + print "Running diff-import for " + self.name + self.updateGeneric(self.parameters, diff_import_file) + self.executeStatements(self.postDiffStatements) + +class MultipolygonSlimTestCase(BaseSlimTestCase): + + def __init__(self, name, parameters, initialStatements, postDiffStatements): + BaseSlimTestCase.__init__(self) + self.name = name + self.parameters = parameters + self.initialStatements = initialStatements + self.postDiffStatements = postDiffStatements + + def setUp(self): + self.setUpGeneric(self.parameters, multipoly_import_file) + + + def runTest(self): + print "****************************************" + print "Running initial import for " + self.name + self.executeStatements(self.initialStatements) + print "Running diff-import for " + self.name + self.updateGeneric(self.parameters, diff_multipoly_import_file) + self.executeStatements(self.postDiffStatements) + + +class BasicGazetteerTestCase(BaseGazetteerTestCase): + + def __init__(self, name, parameters, initialStatements, postDiffStatements): + BaseGazetteerTestCase.__init__(self) + self.name = name + self.parameters = parameters + self.initialStatements = initialStatements + self.postDiffStatements = postDiffStatements + + def setUp(self): + self.setUpGeneric(self.parameters, full_import_file) + + + def runTest(self): + print "****************************************" + print "Running initial import in gazetteer mode for " + self.name + self.executeStatements(self.initialStatements) + print "Running diff-import in gazetteer mode for " + self.name + self.updateGeneric(self.parameters, diff_import_file) + self.executeStatements(self.postDiffStatements) + + +#**************************************************************** +#**************************************************************** +def findContribSql(filename): + + # Try to get base dir for postgres contrib + try: + postgis_base_dir = os.popen('pg_config | grep -m 1 "^INCLUDEDIR ="').read().strip().split(' ')[2].split('/include')[0] + except: + postgis_base_dir = '/usr' + + # Search for the actual sql file + for root, dirs, files in os.walk(postgis_base_dir): + if 'share' in root and 'postgresql' in root and 'contrib' in root and len(fnmatch.filter(files, filename)) > 0: + return '/'.join([root, filename]) + else: + raise Exception('Cannot find %s searching under %s' % (filename, postgis_base_dir)) + +#**************************************************************** +#**************************************************************** +def setupDB(): + print "Setting up test database" + try: + gen_conn=psycopg2.connect("dbname='template1'") + gen_conn.autocommit = True + except Exception as e: + print "I am unable to connect to the database." + exit(1) + + try: + gen_cur = gen_conn.cursor() + except Exception as e: + gen_conn.close() + print "I am unable to connect to the database." + exit(1) + + try: + gen_cur.execute("""DROP DATABASE IF EXISTS \"osm2pgsql-test\"""") + gen_cur.execute("""CREATE DATABASE \"osm2pgsql-test\" WITH ENCODING 'UTF8'""") + except Exception as e: + print "Failed to create osm2pgsql-test db" + e.pgerror + exit(1); + finally: + gen_cur.close() + gen_conn.close() + + try: + test_conn=psycopg2.connect("dbname='osm2pgsql-test'") + test_conn.autocommit = True + except Exception as e: + print "I am unable to connect to the database." + e + exit(1) + + try: + test_cur = test_conn.cursor() + except Exception as e: + print "I am unable to connect to the database." + e + gen_conn.close() + exit(1) + + try: + + # Check the tablespace + try: + global created_tablespace + created_tablespace = 0 + + test_cur.execute("""SELECT spcname FROM pg_tablespace WHERE spcname = 'tablespacetest'""") + if test_cur.fetchone(): + print "We already have a tablespace, can use that" + else: + print "The test needs a temporary tablespace to run in, but it does not exist. Please create the temporary tablespace. On Linux, you can do this by running:" + print " sudo mkdir -p /tmp/psql-tablespace" + print " sudo /bin/chown postgres.postgres tmp/psql-tablespace" + print " psql -c \"CREATE TABLESPACE tablespacetest LOCATION '/tmp/psql-tablespace'\" postgres" + exit(77) + except Exception as e: + print "Failed to create directory for tablespace" + str(e) + + # Check for postgis + try: + test_cur.execute("""CREATE EXTENSION postgis;""") + except: + test_conn.rollback() + + pgis = findContribSql('postgis.sql') + pgscript = open(pgis).read() + test_cur.execute(pgscript) + + srs = findContribSql('spatial_ref_sys.sql') + pgscript = open(srs).read() + test_cur.execute(pgscript) + + # Check for hstore support + try: + test_cur.execute("""CREATE EXTENSION hstore;""") + except Exception as e: + hst = findContribSql('hstore.sql') + pgscript = open(hst).read() + test_cur.execute(pgscript) + + finally: + test_cur.close() + test_conn.close() + +def tearDownDB(): + print "Cleaning up test database" + try: + gen_conn=psycopg2.connect("dbname='template1'") + gen_conn.autocommit = True + gen_cur = gen_conn.cursor() + except Exception as e: + print "I am unable to connect to the database." + exit(1) + + try: + gen_cur.execute("""DROP DATABASE IF EXISTS \"osm2pgsql-test\"""") + if (created_tablespace == 1): + gen_cur.execute("""DROP TABLESPACE IF EXISTS \"tablespacetest\"""") + except Exception as e: + print "Failed to clean up osm2pgsql-test db" + e.pgerror + exit(1); + + gen_cur.close() + gen_conn.close() + if (created_tablespace == 1): + returncode = subprocess.call(["/usr/bin/sudo", "/bin/rmdir", "/tmp/psql-tablespace"]) + + +if __name__ == "__main__": + + from optparse import OptionParser + + parser = OptionParser() + parser.add_option("-f", dest="osm_file", action="store", metavar="FILE", + default=full_import_file, + help="Import a specific osm file [default=%default]") + (options, args) = parser.parse_args() + + if options.osm_file: + full_import_file = options.osm_file + + +ts2 = CompleteTestSuite() +success = False +try: + setupDB() + runner = unittest.TextTestRunner() + result = runner.run(ts2) + success = result.wasSuccessful() + +finally: + tearDownDB() + +if success: + print "All tests passed :-)" + exit(0) +else: + print "Some tests failed :-(" + exit(1) diff -Nru osm2pgsql-0.82.0/tests/regression-test.sh osm2pgsql-0.88.1/tests/regression-test.sh --- osm2pgsql-0.82.0/tests/regression-test.sh 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/tests/regression-test.sh 2015-08-15 05:28:11.000000000 +0000 @@ -1,159 +1,5 @@ -#!/bin/bash -set -e - -trap errorhandler ERR - -errorhandler(){ - echo "!!!!!!TEST failed, please check results!!!!!!" - exit $status -} - -planetfile=$1 -planetdiff=$2 -test_output=`dirname $0`/test_output_$$ - -function setup_db { - echo "" - echo "Initialising test db" - dropdb osm2pgsql-test > /dev/null || true - createdb -E UTF8 osm2pgsql-test - psql -f /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql -d osm2pgsql-test > /dev/null - psql -f /usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql -d osm2pgsql-test > /dev/null - psql -c "CREATE EXTENSION hstore;" -d osm2pgsql-test &> /dev/null - sudo rm -rf /tmp/psql-tablespace || true - mkdir /tmp/psql-tablespace - sudo chown postgres.postgres /tmp/psql-tablespace - psql -q -c "DROP TABLESPACE tablespacetest" -d osm2pgsql-test > /dev/null || true - psql -c "CREATE TABLESPACE tablespacetest LOCATION '/tmp/psql-tablespace'" -d osm2pgsql-test -} - -function teardown_db { - dropdb osm2pgsql-test #To remove any objects that might still be in the table space - psql -c "DROP TABLESPACE tablespacetest" -d postgres - sudo rm -rf /tmp/psql-tablespace - rm -f $test_output $test_output.* - dropdb osm2pgsql-test - -} - -function psql_test { - ( echo -n "$1"; psql -c "$2" -t -d osm2pgsql-test ) | tee -a $test_output.tmp -} - -function reset_results { - rm -f $test_output $test_output.* -} - -function compare_results { - if [ ! -r $test_output ]; then - mv $test_output.tmp $test_output - elif diff $test_output $test_output.tmp >/dev/null; then - rm $test_output.tmp - else - errorhandler - fi -} - -function test_osm2pgsql_slim { - trap errorhandler ERR - echo "" - echo "" - echo "@@@Testing osm2pgsql in slim mode with the following parameters: \"" $1 "\"@@@" - setup_db - - dbprefix=${2:-planet_osm} - - ./osm2pgsql --slim --create -d osm2pgsql-test $1 $planetfile - psql_test "Number of points imported" "SELECT count(*) FROM ${dbprefix}_point;" - psql_test "Number of lines imported" "SELECT count(*) FROM ${dbprefix}_line;" - psql_test "Number of roads imported" "SELECT count(*) FROM ${dbprefix}_roads;" - psql_test "Number of polygon imported" "SELECT count(*) FROM ${dbprefix}_polygon;" - psql_test "Number of nodes imported" "SELECT count(*) FROM ${dbprefix}_nodes;" - psql_test "Number of ways imported" "SELECT count(*) FROM ${dbprefix}_ways;" - psql_test "Number of relations imported" "SELECT count(*) FROM ${dbprefix}_rels;" - - echo "***Testing osm2pgsql diff import with the following parameters: \"" $1 "\"***" - ./osm2pgsql --slim --append -d osm2pgsql-test $1 $planetdiff - psql_test "Number of points imported" "SELECT count(*) FROM ${dbprefix}_point;" - psql_test "Number of lines imported" "SELECT count(*) FROM ${dbprefix}_line;" - psql_test "Number of roads imported" "SELECT count(*) FROM ${dbprefix}_roads;" - psql_test "Number of polygon imported" "SELECT count(*) FROM ${dbprefix}_polygon;" - psql_test "Number of nodes imported" "SELECT count(*) FROM ${dbprefix}_nodes;" - psql_test "Number of ways imported" "SELECT count(*) FROM ${dbprefix}_ways;" - psql_test "Number of relations imported" "SELECT count(*) FROM ${dbprefix}_rels;" - compare_results -} - -function test_osm2pgsql_gazetteer { - trap errorhandler ERR - echo "" - echo "" - echo "@@@Testing osm2pgsql in gazetteer mode with the following parameters: \"" $1 "\"@@@" - setup_db - - dbprefix=${2:-planet_osm} - - ./osm2pgsql --slim --create -l -O gazetteer -d osm2pgsql-test $1 $planetfile - psql_test "Number of places imported" "SELECT count(*) FROM place;" - psql_test "Number of nodes imported" "SELECT count(*) FROM ${dbprefix}_nodes;" - psql_test "Number of ways imported" "SELECT count(*) FROM ${dbprefix}_ways;" - psql_test "Number of relations imported" "SELECT count(*) FROM ${dbprefix}_rels;" - - echo "***Testing osm2pgsql diff import with the following parameters: \"" $1 "\"***" - ./osm2pgsql --slim --append -l -O gazetteer -d osm2pgsql-test $1 $planetdiff - psql_test "Number of places imported" "SELECT count(*) FROM place;" - psql_test "Number of nodes imported" "SELECT count(*) FROM ${dbprefix}_nodes;" - psql_test "Number of ways imported" "SELECT count(*) FROM ${dbprefix}_ways;" - psql_test "Number of relations imported" "SELECT count(*) FROM ${dbprefix}_rels;" - compare_results -} - -function test_osm2pgsql_nonslim { - trap errorhandler ERR - echo "" - echo "" - echo "@@@Testing osm2pgsql with the following parameters: \"" $1 "\"@@@" - setup_db - ./osm2pgsql --create -d osm2pgsql-test $1 $planetfile - psql_test "Number of points imported" "SELECT count(*) FROM planet_osm_point;" - psql_test "Number of lines imported" "SELECT count(*) FROM planet_osm_line;" - psql_test "Number of roads imported" "SELECT count(*) FROM planet_osm_roads;" - psql_test "Number of polygon imported" "SELECT count(*) FROM planet_osm_polygon;" - compare_results -} - - -test_osm2pgsql_nonslim "-S default.style -C 100" -test_osm2pgsql_nonslim "-S default.style -C 100" -echo ========== OK SO FAR ============= -test_osm2pgsql_nonslim "-S default.style -l -C 100" -test_osm2pgsql_nonslim "--slim --drop -S default.style -C 100" -reset_results - -echo ========== NOW DOING SLIM ============= -test_osm2pgsql_slim "-S default.style -C 100" -test_osm2pgsql_slim "-S default.style -l -C 100" -test_osm2pgsql_slim "-k -S default.style -C 100" -test_osm2pgsql_slim "-j -S default.style -C 100" -test_osm2pgsql_slim "-K -S default.style -C 100" -test_osm2pgsql_slim "-x -S default.style -C 100" -test_osm2pgsql_slim "-p planet_osm2 -S default.style -C 100" "planet_osm2" -test_osm2pgsql_slim "--bbox -90.0,-180.0,90.0,180.0 -S default.style -C 100" -test_osm2pgsql_slim "--number-processes 6 -S default.style -C 100" -test_osm2pgsql_slim "-I -S default.style -C 100" -test_osm2pgsql_slim "-e 16:16 -S default.style -C 100" -test_osm2pgsql_slim "--number-processes 6 -e 16:16 -S default.style -C 100" -test_osm2pgsql_slim "-S default.style -C 100 -i tablespacetest" -test_osm2pgsql_slim "-S default.style -C 100 --tablespace-main-data tablespacetest" -test_osm2pgsql_slim "-S default.style -C 100 --tablespace-main-index tablespacetest" -test_osm2pgsql_slim "-S default.style -C 100 --tablespace-slim-data tablespacetest" -test_osm2pgsql_slim "-S default.style -C 100 --tablespace-slim-index tablespacetest" -reset_results - -#test_osm2pgsql_gazetteer "-C 100" -#test_osm2pgsql_gazetteer "--bbox -90.0,-180.0,90.0,180.0 -C 100" - -teardown_db - - +#!/usr/bin/env sh +tests/regression-test.py -f tests/liechtenstein-2013-08-03.osm.pbf && +tests/regression-test.py -f tests/liechtenstein-2013-08-03.osm.bz2 && +echo "Tests passed :-)" diff -Nru osm2pgsql-0.82.0/tests/test-expire-tiles.cpp osm2pgsql-0.88.1/tests/test-expire-tiles.cpp --- osm2pgsql-0.82.0/tests/test-expire-tiles.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-expire-tiles.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,366 @@ +#include "expire-tiles.hpp" +#include "options.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#define EARTH_CIRCUMFERENCE (40075016.68) + +namespace { + +void run_test(const char* test_name, void (*testfunc)()) +{ + try + { + fprintf(stderr, "%s\n", test_name); + testfunc(); + } + catch(const std::exception& e) + { + fprintf(stderr, "%s\n", e.what()); + fprintf(stderr, "FAIL\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "PASS\n"); +} +#define RUN_TEST(x) run_test(#x, &(x)) +#define ASSERT_EQ(a, b) { if (!((a) == (b))) { throw std::runtime_error((boost::format("Expecting %1% == %2%, but %3% != %4%") % #a % #b % (a) % (b)).str()); } } + +struct xyz { + int z, x, y; + xyz(int z_, int x_, int y_) : z(z_), x(x_), y(y_) {} + bool operator==(const xyz &other) const { + return ((z == other.z) && + (x == other.x) && + (y == other.y)); + } + bool operator<(const xyz &other) const { + return ((z < other.z) || + ((z == other.z) && + ((x < other.x) || + ((x == other.x) && + (y < other.y))))); + } + void to_bbox(double &x0, double &y0, + double &x1, double &y1) const { + const double datum = 0.5 * (1 << z); + const double scale = EARTH_CIRCUMFERENCE / (1 << z); + x0 = (x - datum) * scale; + y0 = (datum - (y + 1)) * scale; + x1 = ((x + 1) - datum) * scale; + y1 = (datum - y) * scale; + } + void to_centroid(double &x0, double &y0) const { + const double datum = 0.5 * (1 << z); + const double scale = EARTH_CIRCUMFERENCE / (1 << z); + x0 = ((x + 0.5) - datum) * scale; + y0 = (datum - (y + 0.5)) * scale; + } +}; + +std::ostream &operator<<(std::ostream &out, const xyz &tile) { + out << tile.z << "/" << tile.x << "/" << tile.y; + return out; +} + +struct tile_output_set : public expire_tiles::tile_output { + tile_output_set() {} + + virtual ~tile_output_set() {} + + virtual void output_dirty_tile(int x, int y, int zoom, int min_zoom) { + int y_min, x_iter, y_iter, x_max, y_max, out_zoom, zoom_diff; + + if (zoom > min_zoom) out_zoom = zoom; + else out_zoom = min_zoom; + zoom_diff = out_zoom - zoom; + y_min = y << zoom_diff; + x_max = (x + 1) << zoom_diff; + y_max = (y + 1) << zoom_diff; + for (x_iter = x << zoom_diff; x_iter < x_max; x_iter++) { + for (y_iter = y_min; y_iter < y_max; y_iter++) { + m_tiles.insert(xyz(out_zoom, x_iter, y_iter)); + } + } + } + + std::set m_tiles; +}; + +void test_expire_simple_z1() { + options_t opt; + opt.expire_tiles_zoom = 1; + opt.expire_tiles_zoom_min = 1; + + expire_tiles et(&opt); + tile_output_set set; + + // as big a bbox as possible at the origin to dirty all four + // quadrants of the world. + et.from_bbox(-10000, -10000, 10000, 10000); + et.output_and_destroy(&set); + + ASSERT_EQ(set.m_tiles.size(), 4); + std::set::iterator itr = set.m_tiles.begin(); + ASSERT_EQ(*itr, xyz(1, 0, 0)); ++itr; + ASSERT_EQ(*itr, xyz(1, 0, 1)); ++itr; + ASSERT_EQ(*itr, xyz(1, 1, 0)); ++itr; + ASSERT_EQ(*itr, xyz(1, 1, 1)); ++itr; +} + +void test_expire_simple_z3() { + options_t opt; + opt.expire_tiles_zoom = 3; + opt.expire_tiles_zoom_min = 3; + + expire_tiles et(&opt); + tile_output_set set; + + // as big a bbox as possible at the origin to dirty all four + // quadrants of the world. + et.from_bbox(-10000, -10000, 10000, 10000); + et.output_and_destroy(&set); + + ASSERT_EQ(set.m_tiles.size(), 4); + std::set::iterator itr = set.m_tiles.begin(); + ASSERT_EQ(*itr, xyz(3, 3, 3)); ++itr; + ASSERT_EQ(*itr, xyz(3, 3, 4)); ++itr; + ASSERT_EQ(*itr, xyz(3, 4, 3)); ++itr; + ASSERT_EQ(*itr, xyz(3, 4, 4)); ++itr; +} + +void test_expire_simple_z18() { + options_t opt; + opt.expire_tiles_zoom = 18; + opt.expire_tiles_zoom_min = 18; + + expire_tiles et(&opt); + tile_output_set set; + + // dirty a smaller bbox this time, as at z18 the scale is + // pretty small. + et.from_bbox(-1, -1, 1, 1); + et.output_and_destroy(&set); + + ASSERT_EQ(set.m_tiles.size(), 4); + std::set::iterator itr = set.m_tiles.begin(); + ASSERT_EQ(*itr, xyz(18, 131071, 131071)); ++itr; + ASSERT_EQ(*itr, xyz(18, 131071, 131072)); ++itr; + ASSERT_EQ(*itr, xyz(18, 131072, 131071)); ++itr; + ASSERT_EQ(*itr, xyz(18, 131072, 131072)); ++itr; +} + +std::set generate_random(int zoom, size_t count) { + size_t num = 0; + std::set set; + const int coord_mask = (1 << zoom) - 1; + + while (num < count) { + xyz item(zoom, rand() & coord_mask, rand() & coord_mask); + if (set.count(item) == 0) { + set.insert(item); + ++num; + } + } + + return set; +} + +void assert_tilesets_equal(const std::set &a, + const std::set &b) { + ASSERT_EQ(a.size(), b.size()); + std::set::const_iterator a_itr = a.begin(); + std::set::const_iterator b_itr = b.begin(); + while ((a_itr != a.end()) && + (b_itr != b.end())) { + ASSERT_EQ(*a_itr, *b_itr); + ++a_itr; + ++b_itr; + } +} + +void expire_centroids(const std::set &check_set, + expire_tiles &et) { + for (std::set::const_iterator itr = check_set.begin(); + itr != check_set.end(); ++itr) { + double x0 = 0.0, y0 = 0.0; + itr->to_centroid(x0, y0); + et.from_bbox(x0, y0, x0, y0); + } +} + +// tests that expiring a set of tile centroids means that +// those tiles get expired. +void test_expire_set() { + options_t opt; + int zoom = 18; + opt.expire_tiles_zoom = zoom; + opt.expire_tiles_zoom_min = zoom; + + for (int i = 0; i < 100; ++i) { + expire_tiles et(&opt); + tile_output_set set; + + std::set check_set = generate_random(zoom, 100); + expire_centroids(check_set, et); + + et.output_and_destroy(&set); + + assert_tilesets_equal(set.m_tiles, check_set); + } +} + +// this tests that, after expiring a random set of tiles +// in one expire_tiles object and a different set in +// another, when they are merged together they are the +// same as if the union of the sets of tiles had been +// expired. +void test_expire_merge() { + options_t opt; + int zoom = 18; + opt.expire_tiles_zoom = zoom; + opt.expire_tiles_zoom_min = zoom; + + for (int i = 0; i < 100; ++i) { + expire_tiles et(&opt), et1(&opt), et2(&opt); + tile_output_set set; + + std::set check_set1 = generate_random(zoom, 100); + expire_centroids(check_set1, et1); + + std::set check_set2 = generate_random(zoom, 100); + expire_centroids(check_set2, et2); + + et.merge_and_destroy(et1); + et.merge_and_destroy(et2); + + std::set check_set; + std::set_union(check_set1.begin(), check_set1.end(), + check_set2.begin(), check_set2.end(), + std::inserter(check_set, check_set.end())); + + et.output_and_destroy(&set); + + assert_tilesets_equal(set.m_tiles, check_set); + } +} + +// tests that merging two identical sets results in +// the same set. this guarantees that we check some +// pathways of the merging which possibly could be +// skipped by the random tile set in the previous +// test. +void test_expire_merge_same() { + options_t opt; + int zoom = 18; + opt.expire_tiles_zoom = zoom; + opt.expire_tiles_zoom_min = zoom; + + for (int i = 0; i < 100; ++i) { + expire_tiles et(&opt), et1(&opt), et2(&opt); + tile_output_set set; + + std::set check_set = generate_random(zoom, 100); + expire_centroids(check_set, et1); + expire_centroids(check_set, et2); + + et.merge_and_destroy(et1); + et.merge_and_destroy(et2); + + et.output_and_destroy(&set); + + assert_tilesets_equal(set.m_tiles, check_set); + } +} + +// makes sure that we're testing the case where some +// tiles are in both. +void test_expire_merge_overlap() { + options_t opt; + int zoom = 18; + opt.expire_tiles_zoom = zoom; + opt.expire_tiles_zoom_min = zoom; + + for (int i = 0; i < 100; ++i) { + expire_tiles et(&opt), et1(&opt), et2(&opt); + tile_output_set set; + + std::set check_set1 = generate_random(zoom, 100); + expire_centroids(check_set1, et1); + + std::set check_set2 = generate_random(zoom, 100); + expire_centroids(check_set2, et2); + + std::set check_set3 = generate_random(zoom, 100); + expire_centroids(check_set3, et1); + expire_centroids(check_set3, et2); + + et.merge_and_destroy(et1); + et.merge_and_destroy(et2); + + std::set check_set; + std::set_union(check_set1.begin(), check_set1.end(), + check_set2.begin(), check_set2.end(), + std::inserter(check_set, check_set.end())); + std::set_union(check_set1.begin(), check_set1.end(), + check_set3.begin(), check_set3.end(), + std::inserter(check_set, check_set.end())); + + et.output_and_destroy(&set); + + assert_tilesets_equal(set.m_tiles, check_set); + } +} + +// checks that the set union still works when we expire +// large contiguous areas of tiles (i.e: ensure that we +// handle the "complete" flag correctly). +void test_expire_merge_complete() { + options_t opt; + int zoom = 18; + opt.expire_tiles_zoom = zoom; + opt.expire_tiles_zoom_min = zoom; + + for (int i = 0; i < 100; ++i) { + expire_tiles et(&opt), et1(&opt), et2(&opt), et0(&opt); + tile_output_set set, set0; + + // et1&2 are two halves of et0's box + et0.from_bbox(-10000, -10000, 10000, 10000); + et1.from_bbox(-10000, -10000, 0, 10000); + et2.from_bbox( 0, -10000, 10000, 10000); + + et.merge_and_destroy(et1); + et.merge_and_destroy(et2); + + et.output_and_destroy(&set); + et0.output_and_destroy(&set0); + + assert_tilesets_equal(set.m_tiles, set0.m_tiles); + } +} + +} // anonymous namespace + +int main(int argc, char *argv[]) +{ + srand(0); + + //try each test if any fail we will exit + RUN_TEST(test_expire_simple_z1); + RUN_TEST(test_expire_simple_z3); + RUN_TEST(test_expire_simple_z18); + RUN_TEST(test_expire_set); + RUN_TEST(test_expire_merge); + RUN_TEST(test_expire_merge_same); + RUN_TEST(test_expire_merge_overlap); + RUN_TEST(test_expire_merge_complete); + + //passed + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test-middle-flat.cpp osm2pgsql-0.88.1/tests/test-middle-flat.cpp --- osm2pgsql-0.82.0/tests/test-middle-flat.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-middle-flat.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "output-null.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" + +#include +#include +#include + +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +/* This is basically the same as test-middle-pgsql, but with flat nodes. */ + +void run_tests(options_t options, const std::string cache_type) { + options.append = 0; + options.create = 1; + options.flat_node_cache_enabled = true; + // flat nodes truncates the file each time it's started, so we can reuse the same file + options.flat_node_file = boost::optional("tests/test_middle_flat.flat.nodes.bin"); + + { + middle_pgsql_t mid_pgsql; + output_null_t out_test(&mid_pgsql, options); + + mid_pgsql.start(&options); + + if (test_node_set(&mid_pgsql) != 0) { throw std::runtime_error("test_node_set failed."); } + + mid_pgsql.commit(); + mid_pgsql.stop(); + } + { + middle_pgsql_t mid_pgsql; + output_null_t out_test(&mid_pgsql, options); + + mid_pgsql.start(&options); + + if (test_nodes_comprehensive_set(&mid_pgsql) != 0) { throw std::runtime_error("test_nodes_comprehensive_set failed."); } + + mid_pgsql.commit(); + mid_pgsql.stop(); + } + /* This should work, but doesn't. More tests are needed that look at updates + without the complication of ways. + */ +/* { + middle_pgsql_t mid_pgsql; + output_null_t out_test(&mid_pgsql, options); + + mid_pgsql.start(&options); + mid_pgsql.commit(); + mid_pgsql.stop(); + // Switch to append mode because this tests updates + options.append = 1; + options.create = 0; + mid_pgsql.start(&options); + if (test_way_set(&mid_pgsql) != 0) { throw std::runtime_error("test_way_set failed."); } + + mid_pgsql.commit(); + mid_pgsql.stop(); + }*/ +} +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + options_t options; + options.conninfo = db->conninfo().c_str(); + options.scale = 10000000; + options.cache = 1; + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + + options.alloc_chunkwise = ALLOC_SPARSE | ALLOC_DENSE; // what you get with optimized + run_tests(options, "optimized"); + options.alloc_chunkwise = ALLOC_SPARSE; + run_tests(options, "sparse"); + + options.alloc_chunkwise = ALLOC_DENSE; + run_tests(options, "dense"); + + options.alloc_chunkwise = ALLOC_DENSE | ALLOC_DENSE_CHUNK; // what you get with chunk + run_tests(options, "chunk"); + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + return 1; + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + return 1; + } + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test-middle-pgsql.cpp osm2pgsql-0.88.1/tests/test-middle-pgsql.cpp --- osm2pgsql-0.82.0/tests/test-middle-pgsql.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-middle-pgsql.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,103 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "output-null.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" + +#include +#include +#include + +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void run_tests(options_t options, const std::string cache_type) { + options.append = 0; + options.create = 1; + { + middle_pgsql_t mid_pgsql; + output_null_t out_test(&mid_pgsql, options); + + mid_pgsql.start(&options); + + if (test_node_set(&mid_pgsql) != 0) { throw std::runtime_error("test_node_set failed."); } + + mid_pgsql.commit(); + mid_pgsql.stop(); + } + { + middle_pgsql_t mid_pgsql; + output_null_t out_test(&mid_pgsql, options); + + mid_pgsql.start(&options); + + if (test_nodes_comprehensive_set(&mid_pgsql) != 0) { throw std::runtime_error("test_nodes_comprehensive_set failed."); } + + mid_pgsql.commit(); + mid_pgsql.stop(); + } + { + middle_pgsql_t mid_pgsql; + output_null_t out_test(&mid_pgsql, options); + + mid_pgsql.start(&options); + mid_pgsql.commit(); + mid_pgsql.stop(); + // Switch to append mode because this tests updates + options.append = 1; + options.create = 0; + mid_pgsql.start(&options); + if (test_way_set(&mid_pgsql) != 0) { throw std::runtime_error("test_way_set failed."); } + + mid_pgsql.commit(); + mid_pgsql.stop(); + } +} +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + options_t options; + options.conninfo = db->conninfo().c_str(); + options.scale = 10000000; + options.cache = 1; + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + + options.alloc_chunkwise = ALLOC_SPARSE | ALLOC_DENSE; // what you get with optimized + run_tests(options, "optimized"); + options.alloc_chunkwise = ALLOC_SPARSE; + run_tests(options, "sparse"); + + options.alloc_chunkwise = ALLOC_DENSE; + run_tests(options, "dense"); + + options.alloc_chunkwise = ALLOC_DENSE | ALLOC_DENSE_CHUNK; // what you get with chunk + run_tests(options, "chunk"); + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + return 1; + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + return 1; + } + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test-middle-ram.cpp osm2pgsql-0.88.1/tests/test-middle-ram.cpp --- osm2pgsql-0.82.0/tests/test-middle-ram.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-middle-ram.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "output-null.hpp" +#include "options.hpp" +#include "middle-ram.hpp" + +#include "tests/middle-tests.hpp" + +void run_tests(const options_t options, const std::string cache_type) { + { + middle_ram_t mid_ram; + output_null_t out_test(&mid_ram, options); + + mid_ram.start(&options); + + if (test_node_set(&mid_ram) != 0) { throw std::runtime_error("test_node_set failed with " + cache_type + " cache."); } + mid_ram.commit(); + mid_ram.stop(); + } + { + middle_ram_t mid_ram; + output_null_t out_test(&mid_ram, options); + + mid_ram.start(&options); + + if (test_nodes_comprehensive_set(&mid_ram) != 0) { throw std::runtime_error("test_nodes_comprehensive_set failed with " + cache_type + " cache."); } + mid_ram.commit(); + mid_ram.stop(); + } + { + middle_ram_t mid_ram; + output_null_t out_test(&mid_ram, options); + + mid_ram.start(&options); + + if (test_way_set(&mid_ram) != 0) { throw std::runtime_error("test_way_set failed with " + cache_type + " cache."); } + mid_ram.commit(); + mid_ram.stop(); + } +} + +int main(int argc, char *argv[]) { + try { + options_t options; + options.scale = 10000000; + options.cache = 1; // Non-zero cache is needed to test + + options.alloc_chunkwise = ALLOC_SPARSE | ALLOC_DENSE; // what you get with optimized + run_tests(options, "optimized"); + + options.alloc_chunkwise = ALLOC_SPARSE; + run_tests(options, "sparse"); + + options.alloc_chunkwise = ALLOC_DENSE; + run_tests(options, "dense"); + + options.alloc_chunkwise = ALLOC_DENSE | ALLOC_DENSE_CHUNK; // what you get with chunk + run_tests(options, "chunk"); + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + return 1; + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + return 1; + } + + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test_multipolygon_diff.osc osm2pgsql-0.88.1/tests/test_multipolygon_diff.osc --- osm2pgsql-0.82.0/tests/test_multipolygon_diff.osc 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_multipolygon_diff.osc 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,193 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test_multipolygon.osm osm2pgsql-0.88.1/tests/test_multipolygon.osm --- osm2pgsql-0.82.0/tests/test_multipolygon.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_multipolygon.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,1508 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test_multipolygon_postdiff.osm osm2pgsql-0.88.1/tests/test_multipolygon_postdiff.osm --- osm2pgsql-0.82.0/tests/test_multipolygon_postdiff.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_multipolygon_postdiff.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,987 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test-output-multi-line.cpp osm2pgsql-0.88.1/tests/test-output-multi-line.cpp --- osm2pgsql-0.82.0/tests/test-output-multi-line.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-multi-line.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,122 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "output-multi.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options; + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.slim = 1; + + boost::shared_ptr processor = + geometry_processor::create("line", &options); + + export_list columns; + { taginfo info; info.name = "highway"; info.type = "text"; columns.add(OSMTYPE_WAY, info); } + + boost::shared_ptr out_test(new output_multi_t("foobar_highways", processor, columns, mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'foobar_highways'"); + check_count(test_conn, 2753, "select count(*) from foobar_highways"); + + //check that we have the right spread + check_count(test_conn, 13, "select count(*) from foobar_highways where highway='bridleway'"); + check_count(test_conn, 3, "select count(*) from foobar_highways where highway='construction'"); + check_count(test_conn, 96, "select count(*) from foobar_highways where highway='cycleway'"); + check_count(test_conn, 249, "select count(*) from foobar_highways where highway='footway'"); + check_count(test_conn, 18, "select count(*) from foobar_highways where highway='living_street'"); + check_count(test_conn, 171, "select count(*) from foobar_highways where highway='path'"); + check_count(test_conn, 6, "select count(*) from foobar_highways where highway='pedestrian'"); + check_count(test_conn, 81, "select count(*) from foobar_highways where highway='primary'"); + check_count(test_conn, 842, "select count(*) from foobar_highways where highway='residential'"); + check_count(test_conn, 3, "select count(*) from foobar_highways where highway='road'"); + check_count(test_conn, 90, "select count(*) from foobar_highways where highway='secondary'"); + check_count(test_conn, 1, "select count(*) from foobar_highways where highway='secondary_link'"); + check_count(test_conn, 352, "select count(*) from foobar_highways where highway='service'"); + check_count(test_conn, 34, "select count(*) from foobar_highways where highway='steps'"); + check_count(test_conn, 33, "select count(*) from foobar_highways where highway='tertiary'"); + check_count(test_conn, 597, "select count(*) from foobar_highways where highway='track'"); + check_count(test_conn, 164, "select count(*) from foobar_highways where highway='unclassified'"); + return 0; + + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/tests/test-output-multi-line-storage.cpp osm2pgsql-0.88.1/tests/test-output-multi-line-storage.cpp --- osm2pgsql-0.82.0/tests/test-output-multi-line-storage.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-multi-line-storage.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "middle.hpp" +#include "output-multi.hpp" +#include "options.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + options_t options; + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.slim = 1; + + options.projection.reset(new reprojection(PROJ_LATLONG)); + + options.output_backend = "multi"; + options.style = "tests/test_output_multi_line_trivial.style.json"; + + //setup the front (input) + parse_delegate_t parser(options.extra_attributes, options.bbox, options.projection); + + //setup the middle + boost::shared_ptr middle = middle_t::create_middle(options.slim); + + //setup the backend (output) + std::vector > outputs = output_t::create_outputs(middle.get(), options); + + //let osmdata orchestrate between the middle and the outs + osmdata_t osmdata(middle, outputs); + + osmdata.start(); + + if (parser.streamFile("libxml2", "tests/test_output_multi_line_storage.osm", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/test_output_multi_line_storage.osm'."); + } + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'test_line'"); + check_count(test_conn, 3, "select count(*) from test_line"); + + //check that we have the number of vertexes in each linestring + check_count(test_conn, 3, "SELECT ST_NumPoints(way) FROM test_line WHERE osm_id = 1"); + check_count(test_conn, 2, "SELECT ST_NumPoints(way) FROM test_line WHERE osm_id = 2"); + check_count(test_conn, 2, "SELECT ST_NumPoints(way) FROM test_line WHERE osm_id = 3"); + + check_count(test_conn, 3, "SELECT COUNT(*) FROM test_line WHERE foo = 'bar'"); + return 0; + + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_line_storage.osm osm2pgsql-0.88.1/tests/test_output_multi_line_storage.osm --- osm2pgsql-0.82.0/tests/test_output_multi_line_storage.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_line_storage.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_line_trivial.lua osm2pgsql-0.88.1/tests/test_output_multi_line_trivial.lua --- osm2pgsql-0.82.0/tests/test_output_multi_line_trivial.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_line_trivial.lua 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,10 @@ +function drop_all (...) + return 1, {} +end + +-- A generic way to process ways, given a function which determines if tags are interesting +-- Takes an optional function to process tags. Always says it's a polygon if there's matching tags +function test_ways (kv, num_keys) + tags = {["foo"] = "bar"} + return 0, tags, 1, 0 +end diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_line_trivial.style.json osm2pgsql-0.88.1/tests/test_output_multi_line_trivial.style.json --- osm2pgsql-0.82.0/tests/test_output_multi_line_trivial.style.json 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_line_trivial.style.json 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,14 @@ +[ + { + "name": "test_line", + "type": "line", + "tagtransform": "tests/test_output_multi_line_trivial.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "test_ways", + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "foo", "type": "text"} + ] + } +] diff -Nru osm2pgsql-0.82.0/tests/test-output-multi-point.cpp osm2pgsql-0.88.1/tests/test-output-multi-point.cpp --- osm2pgsql-0.82.0/tests/test-output-multi-point.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-multi-point.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "output-multi.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options; + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + + boost::shared_ptr processor = + geometry_processor::create("point", &options); + + export_list columns; + { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(OSMTYPE_NODE, info); } + + boost::shared_ptr out_test(new output_multi_t("foobar_amenities", processor, columns, mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + check_count(test_conn, 1, + "select count(*) from pg_catalog.pg_class " + "where relname = 'foobar_amenities'"); + + check_count(test_conn, 244, + "select count(*) from foobar_amenities"); + + check_count(test_conn, 36, + "select count(*) from foobar_amenities where amenity='parking'"); + + check_count(test_conn, 34, + "select count(*) from foobar_amenities where amenity='bench'"); + + check_count(test_conn, 1, + "select count(*) from foobar_amenities where amenity='vending_machine'"); + + return 0; + + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/tests/test-output-multi-point-multi-table.cpp osm2pgsql-0.88.1/tests/test-output-multi-point-multi-table.cpp --- osm2pgsql-0.82.0/tests/test-output-multi-point-multi-table.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-multi-point-multi-table.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,144 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "output-multi.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + if (PQresultStatus(res->get()) != PGRES_TUPLES_OK) { + throw std::runtime_error((boost::format("Query ERROR running %1%: %2%") + % query % PQresultErrorMessage(res->get())).str()); + } + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options; + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + + export_list columns; + { taginfo info; info.name = "amenity"; info.type = "text"; columns.add(OSMTYPE_NODE, info); } + + std::vector > outputs; + + // let's make lots of tables! + for (int i = 0; i < 10; ++i) { + std::string name = (boost::format("foobar_%d") % i).str(); + + boost::shared_ptr processor = + geometry_processor::create("point", &options); + + boost::shared_ptr out_test(new output_multi_t(name, processor, columns, mid_pgsql.get(), options)); + + outputs.push_back(out_test); + } + + osmdata_t osmdata(mid_pgsql, outputs); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + for (int i = 0; i < 10; ++i) { + std::string name = (boost::format("foobar_%d") % i).str(); + + check_count(test_conn, 1, + (boost::format("select count(*) from pg_catalog.pg_class " + "where relname = 'foobar_%d'") + % i).str()); + + check_count(test_conn, 244, + (boost::format("select count(*) from foobar_%d") + % i).str()); + + check_count(test_conn, 36, + (boost::format("select count(*) from foobar_%d " + "where amenity='parking'") + % i).str()); + + check_count(test_conn, 34, + (boost::format("select count(*) from foobar_%d " + "where amenity='bench'") + % i).str()); + + check_count(test_conn, 1, + (boost::format("select count(*) from foobar_%d " + "where amenity='vending_machine'") + % i).str()); + } + + return 0; + + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/tests/test-output-multi-polygon.cpp osm2pgsql-0.88.1/tests/test-output-multi-polygon.cpp --- osm2pgsql-0.82.0/tests/test-output-multi-polygon.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-multi-polygon.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "output-multi.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options; + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + + boost::shared_ptr processor = geometry_processor::create("polygon", &options); + + export_list columns; + { taginfo info; info.name = "building"; info.type = "text"; columns.add(OSMTYPE_WAY, info); } + + boost::shared_ptr out_test(new output_multi_t("foobar_buildings", processor, columns, mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'foobar_buildings'"); + check_count(test_conn, 0, "select count(*) from foobar_buildings where building is null"); + check_count(test_conn, 3723, "select count(*) from foobar_buildings"); + + //check that we have the right spread + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='barn'"); + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='chapel'"); + check_count(test_conn, 5, "select count(*) from foobar_buildings where building='church'"); + check_count(test_conn, 3, "select count(*) from foobar_buildings where building='commercial'"); + check_count(test_conn, 6, "select count(*) from foobar_buildings where building='farm'"); + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='garage'"); + check_count(test_conn, 2, "select count(*) from foobar_buildings where building='glasshouse'"); + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='greenhouse'"); + check_count(test_conn, 153, "select count(*) from foobar_buildings where building='house'"); + check_count(test_conn, 4, "select count(*) from foobar_buildings where building='hut'"); + check_count(test_conn, 8, "select count(*) from foobar_buildings where building='industrial'"); + check_count(test_conn, 200, "select count(*) from foobar_buildings where building='residential'"); + check_count(test_conn, 6, "select count(*) from foobar_buildings where building='roof'"); + check_count(test_conn, 4, "select count(*) from foobar_buildings where building='school'"); + check_count(test_conn, 2, "select count(*) from foobar_buildings where building='station'"); + check_count(test_conn, 3, "select count(*) from foobar_buildings where building='warehouse'"); + check_count(test_conn, 3323, "select count(*) from foobar_buildings where building='yes'"); + return 0; + + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/tests/test-output-multi-poly-trivial.cpp osm2pgsql-0.88.1/tests/test-output-multi-poly-trivial.cpp --- osm2pgsql-0.82.0/tests/test-output-multi-poly-trivial.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-multi-poly-trivial.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "middle.hpp" +#include "output-multi.hpp" +#include "options.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +void run_osm2pgsql(options_t &options) { + //setup the front (input) + parse_delegate_t parser(options.extra_attributes, options.bbox, options.projection); + + //setup the middle + boost::shared_ptr middle = middle_t::create_middle(options.slim); + + //setup the backend (output) + std::vector > outputs = output_t::create_outputs(middle.get(), options); + + //let osmdata orchestrate between the middle and the outs + osmdata_t osmdata(middle, outputs); + + osmdata.start(); + + if (parser.streamFile("libxml2", "tests/test_output_multi_poly_trivial.osm", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/test_output_multi_poly_trivial.osm'."); + } + + osmdata.stop(); +} + +void check_output_poly_trivial(int enable_multi, std::string conninfo) { + options_t options; + options.conninfo = conninfo.c_str(); + options.num_procs = 1; + options.slim = 1; + options.enable_multi = enable_multi; + + options.projection.reset(new reprojection(PROJ_LATLONG)); + + options.output_backend = "multi"; + options.style = "tests/test_output_multi_poly_trivial.style.json"; + + run_osm2pgsql(options); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(conninfo); + + // expect that the table exists + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'test_poly'"); + + // expect 2 polygons if not in multi(geometry) mode, or 1 if multi(geometry) + // mode is enabled. + if (enable_multi) { + check_count(test_conn, 1, "select count(*) from test_poly"); + check_count(test_conn, 1, "select count(*) from test_poly where foo='bar'"); + check_count(test_conn, 1, "select count(*) from test_poly where bar='baz'"); + + // there should be two 5-pointed polygons in the multipolygon (note that + // it's 5 points including the duplicated first/last point) + check_count(test_conn, 2, "select count(*) from (select (st_dump(way)).geom as way from test_poly) x"); + check_count(test_conn, 5, "select distinct st_numpoints(st_exteriorring(way)) from (select (st_dump(way)).geom as way from test_poly) x"); + + } else { + check_count(test_conn, 2, "select count(*) from test_poly"); + check_count(test_conn, 2, "select count(*) from test_poly where foo='bar'"); + check_count(test_conn, 2, "select count(*) from test_poly where bar='baz'"); + + // although there are 2 rows, they should both be 5-pointed polygons (note + // that it's 5 points including the duplicated first/last point) + check_count(test_conn, 5, "select distinct st_numpoints(st_exteriorring(way)) from test_poly"); + } +} + +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + check_output_poly_trivial(0, db->conninfo()); + check_output_poly_trivial(1, db->conninfo()); + return 0; + + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_poly_trivial.lua osm2pgsql-0.88.1/tests/test_output_multi_poly_trivial.lua --- osm2pgsql-0.82.0/tests/test_output_multi_poly_trivial.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_poly_trivial.lua 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,24 @@ +function drop_all (...) + return 1, {} +end + +function drop_ways (...) + return 1, {}, 0, 0 +end + +function test_rels (kv, num_keys) + tags = {["foo"] = "bar"} + return 0, tags +end + +function test_members (kv, member_tags, roles, num_members) + membersuperseeded = {} + for i = 1, num_members do + membersuperseeded[i] = 0 + end + + tags = kv + tags["bar"] = "baz" + + return 0, tags, membersuperseeded, 0, 0, 0 +end diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_poly_trivial.osm osm2pgsql-0.88.1/tests/test_output_multi_poly_trivial.osm --- osm2pgsql-0.82.0/tests/test_output_multi_poly_trivial.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_poly_trivial.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_poly_trivial.style.json osm2pgsql-0.88.1/tests/test_output_multi_poly_trivial.style.json --- osm2pgsql-0.82.0/tests/test_output_multi_poly_trivial.style.json 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_poly_trivial.style.json 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,15 @@ +[ + { + "name": "test_poly", + "type": "polygon", + "tagtransform": "tests/test_output_multi_poly_trivial.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "drop_ways", + "tagtransform-relation-function": "test_rels", + "tagtransform-relation-member-function": "test_members", + "tags": [ + {"name": "foo", "type": "text"}, + {"name": "bar", "type": "text"} + ] + } +] diff -Nru osm2pgsql-0.82.0/tests/test-output-multi-tags.cpp osm2pgsql-0.88.1/tests/test-output-multi-tags.cpp --- osm2pgsql-0.82.0/tests/test-output-multi-tags.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-multi-tags.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,130 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "middle.hpp" +#include "output-multi.hpp" +#include "options.hpp" +#include "osmdata.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +int main(int argc, char *argv[]) { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + return 77; // <-- code to skip this test. + } + + try { + options_t options; + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.slim = 1; + + options.projection.reset(new reprojection(PROJ_LATLONG)); + + options.output_backend = "multi"; + options.style = "tests/test_output_multi_tags.json"; + + //setup the front (input) + parse_delegate_t parser(options.extra_attributes, options.bbox, options.projection); + + //setup the middle + boost::shared_ptr middle = middle_t::create_middle(options.slim); + + //setup the backend (output) + std::vector > outputs = output_t::create_outputs(middle.get(), options); + + //let osmdata orchestrate between the middle and the outs + osmdata_t osmdata(middle, outputs); + + osmdata.start(); + + if (parser.streamFile("libxml2", "tests/test_output_multi_tags.osm", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/test_output_multi_line_storage.osm'."); + } + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + // Check we got the right tables + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'test_points_1'"); + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'test_points_2'"); + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'test_line_1'"); + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'test_polygon_1'"); + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'test_polygon_2'"); + + // Check we didn't get any extra in the tables + check_count(test_conn, 2, "select count(*) from test_points_1"); + check_count(test_conn, 2, "select count(*) from test_points_2"); + check_count(test_conn, 1, "select count(*) from test_line_1"); + check_count(test_conn, 1, "select count(*) from test_line_2"); + check_count(test_conn, 1, "select count(*) from test_polygon_1"); + check_count(test_conn, 1, "select count(*) from test_polygon_2"); + + // Check that the first table for each type got the right transform + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_points_1 WHERE foo IS NULL and bar = 'n1' AND baz IS NULL"); + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_points_1 WHERE foo IS NULL and bar = 'n2' AND baz IS NULL"); + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_line_1 WHERE foo IS NULL and bar = 'w1' AND baz IS NULL"); + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_polygon_1 WHERE foo IS NULL and bar = 'w2' AND baz IS NULL"); + + // Check that the second table also got the right transform + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_points_2 WHERE foo IS NULL and bar IS NULL AND baz = 'n1'"); + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_points_2 WHERE foo IS NULL and bar IS NULL AND baz = 'n2'"); + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_line_2 WHERE foo IS NULL and bar IS NULL AND baz = 'w1'"); + check_count(test_conn, 1, "SELECT COUNT(*) FROM test_polygon_2 WHERE foo IS NULL and bar IS NULL AND baz = 'w2'"); + + return 0; + + } catch (const std::exception &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + + } catch (...) { + std::cerr << "UNKNOWN ERROR" << std::endl; + } + + return 1; +} diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_tags.json osm2pgsql-0.88.1/tests/test_output_multi_tags.json --- osm2pgsql-0.82.0/tests/test_output_multi_tags.json 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_tags.json 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,99 @@ +/* +These tables provide a test that tag processing works, and that one tag +processing does not clobber another. They are not exhaustive, +particularly with regards to interactions between way/area detection and +relations. +*/ +[ + { + "name": "test_points_1", + "type": "point", + "tagtransform": "tests/test_output_multi_tags.lua", + "tagtransform-node-function": "test_nodes_1", + "tagtransform-way-function": "drop_all", + /* No relations in the file */ + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "foo", "type": "text"}, + {"name": "bar", "type": "text"}, + {"name": "baz", "type": "text"} /* left empty by the transform */ + ] + }, + { + /* This second table will process the same data, but resulting in different tags */ + "name": "test_points_2", + "type": "point", + "tagtransform": "tests/test_output_multi_tags.lua", + "tagtransform-node-function": "test_nodes_2", + "tagtransform-way-function": "drop_all", + /* No relations in the file */ + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "foo", "type": "text"}, + {"name": "bar", "type": "text"}, /* left empty by the transform */ + {"name": "baz", "type": "text"} + ] + }, + { + "name": "test_line_1", + "type": "line", + "tagtransform": "tests/test_output_multi_tags.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "test_line_1", + /* No relations in the file */ + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "foo", "type": "text"}, + {"name": "bar", "type": "text"}, + {"name": "baz", "type": "text"} /* left empty by the transform */ + ] + }, + { + "name": "test_line_2", + "type": "line", + "tagtransform": "tests/test_output_multi_tags.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "test_line_2", + /* No relations in the file */ + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "foo", "type": "text"}, + {"name": "bar", "type": "text"}, /* left empty by the transform */ + {"name": "baz", "type": "text"} + ] + }, + { + "name": "test_polygon_1", + "type": "line", + "tagtransform": "tests/test_output_multi_tags.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "test_polygon_1", + /* No relations in the file */ + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "foo", "type": "text"}, + {"name": "bar", "type": "text"}, + {"name": "baz", "type": "text"} /* left empty by the transform */ + ] + }, + { + "name": "test_polygon_2", + "type": "line", + "tagtransform": "tests/test_output_multi_tags.lua", + "tagtransform-node-function": "drop_all", + "tagtransform-way-function": "test_polygon_2", + /* No relations in the file */ + "tagtransform-relation-function": "drop_all", + "tagtransform-relation-member-function": "drop_all", + "tags": [ + {"name": "foo", "type": "text"}, + {"name": "bar", "type": "text"}, + {"name": "baz", "type": "text"} /* left empty by the transform */ + ] + } +] diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_tags.lua osm2pgsql-0.88.1/tests/test_output_multi_tags.lua --- osm2pgsql-0.82.0/tests/test_output_multi_tags.lua 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_tags.lua 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,63 @@ +function drop_all (...) + return 1, {} +end + +function test_nodes_1 (kv, num_tags) + if kv["foo"] then + tags = {} + tags["bar"] = kv["foo"] + return 0, tags + else + return 1, {} + end +end + +function test_nodes_2 (kv, num_tags) + if kv["foo"] then + tags = {} + tags["baz"] = kv["foo"] + return 0, tags + else + return 1, {} + end +end + +function test_line_1 (kv, num_tags) + if kv["foo"] and kv["area"] == "false" then + tags = {} + tags["bar"] = kv["foo"] + return 0, tags, 0, 0 + else + return 1, {}, 0, 0 + end +end + +function test_line_2 (kv, num_tags) + if kv["foo"] and kv["area"] == "false" then + tags = {} + tags["baz"] = kv["foo"] + return 0, tags, 0, 0 + else + return 1, {}, 0, 0 + end +end + +function test_polygon_1 (kv, num_tags) + if kv["foo"] and kv["area"] == "true" then + tags = {} + tags["bar"] = kv["foo"] + return 0, tags, 0, 0 + else + return 1, {}, 0, 0 + end +end + +function test_polygon_2 (kv, num_tags) + if kv["foo"] and kv["area"] == "true" then + tags = {} + tags["baz"] = kv["foo"] + return 0, tags, 0, 0 + else + return 1, {}, 0, 0 + end +end diff -Nru osm2pgsql-0.82.0/tests/test_output_multi_tags.osm osm2pgsql-0.88.1/tests/test_output_multi_tags.osm --- osm2pgsql-0.82.0/tests/test_output_multi_tags.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_multi_tags.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test-output-pgsql.cpp osm2pgsql-0.88.1/tests/test-output-pgsql.cpp --- osm2pgsql-0.82.0/tests/test-output-pgsql.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-pgsql.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,405 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "output-pgsql.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "middle-ram.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +namespace { + +struct skip_test : public std::exception { + const char *what() { return "Test skipped."; } +}; + +void run_test(const char* test_name, void (*testfunc)()) { + try { + fprintf(stderr, "%s\n", test_name); + testfunc(); + + } catch (const skip_test &) { + exit(77); // <-- code to skip this test. + + } catch (const std::exception& e) { + fprintf(stderr, "%s\n", e.what()); + fprintf(stderr, "FAIL\n"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "PASS\n"); +} +#define RUN_TEST(x) run_test(#x, &(x)) + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +void check_number(pg::conn_ptr &conn, double expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query, " + " but got %1%. Query was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + double num = boost::lexical_cast(numstr); + + // floating point isn't exact, so allow a 0.01% difference + if ((num > 1.0001*expected) || (num < 0.9999*expected)) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % num % query).str()); + } +} + +void assert_has_table(pg::conn_ptr &test_conn, const std::string &table_name) { + std::string query = (boost::format("select count(*) from pg_catalog.pg_class " + "where relname = '%1%'") + % table_name).str(); + + check_count(test_conn, 1, query); +} + +// "simple" test modeled on the basic regression test from +// the python script. this is just to check everything is +// working as expected before we start the complex stuff. +void test_regression_simple() { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + throw skip_test(); + } + + std::string proc_name("test-output-pgsql"), input_file("-"); + char *argv[] = { &proc_name[0], &input_file[0], NULL }; + + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options = options_t::parse(2, argv); + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + options.style = "default.style"; + + boost::shared_ptr out_test(new output_pgsql_t(mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + assert_has_table(test_conn, "osm2pgsql_test_point"); + assert_has_table(test_conn, "osm2pgsql_test_line"); + assert_has_table(test_conn, "osm2pgsql_test_polygon"); + assert_has_table(test_conn, "osm2pgsql_test_roads"); + + check_count(test_conn, 1342, "SELECT count(*) FROM osm2pgsql_test_point"); + check_count(test_conn, 3300, "SELECT count(*) FROM osm2pgsql_test_line"); + check_count(test_conn, 375, "SELECT count(*) FROM osm2pgsql_test_roads"); + check_count(test_conn, 4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + + // Check size of lines + check_number(test_conn, 1696.04, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); + check_number(test_conn, 1151.26, "SELECT ST_Length(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); + + check_number(test_conn, 311.21, "SELECT way_area FROM osm2pgsql_test_polygon WHERE osm_id = 157261342"); + check_number(test_conn, 311.21, "SELECT ST_Area(way) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342"); + check_number(test_conn, 143.81, "SELECT ST_Area(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342"); + + // Check a point's location + check_count(test_conn, 1, "SELECT count(*) FROM osm2pgsql_test_point WHERE ST_DWithin(way, 'SRID=900913;POINT(1062645.12 5972593.4)'::geometry, 0.1)"); +} + +void test_latlong() { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + throw skip_test(); + } + + std::string proc_name("test-output-pgsql"), input_file("-"); + char *argv[] = { &proc_name[0], &input_file[0], NULL }; + + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options = options_t::parse(2, argv); + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + options.style = "default.style"; + + options.projection.reset(new reprojection(PROJ_LATLONG)); + options.scale = (options.projection->get_proj_id() == PROJ_LATLONG) ? 10000000 : 100; + + boost::shared_ptr out_test(new output_pgsql_t(mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + assert_has_table(test_conn, "osm2pgsql_test_point"); + assert_has_table(test_conn, "osm2pgsql_test_line"); + assert_has_table(test_conn, "osm2pgsql_test_polygon"); + assert_has_table(test_conn, "osm2pgsql_test_roads"); + + check_count(test_conn, 1342, "SELECT count(*) FROM osm2pgsql_test_point"); + check_count(test_conn, 3298, "SELECT count(*) FROM osm2pgsql_test_line"); + check_count(test_conn, 374, "SELECT count(*) FROM osm2pgsql_test_roads"); + check_count(test_conn, 4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); + + // Check size of lines + check_number(test_conn, 0.0105343, "SELECT ST_Length(way) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); + check_number(test_conn, 1151.26, "SELECT ST_Length(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_line WHERE osm_id = 44822682"); + + check_number(test_conn, 1.70718e-08, "SELECT way_area FROM osm2pgsql_test_polygon WHERE osm_id = 157261342"); + check_number(test_conn, 1.70718e-08, "SELECT ST_Area(way) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342"); + check_number(test_conn, 143.845, "SELECT ST_Area(ST_Transform(way,4326)::geography) FROM osm2pgsql_test_polygon WHERE osm_id = 157261342"); + + // Check a point's location + check_count(test_conn, 1, "SELECT count(*) FROM osm2pgsql_test_point WHERE ST_DWithin(way, 'SRID=4326;POINT(9.5459035 47.1866494)'::geometry, 0.00001)"); +} + + +void test_area_way_simple() { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + throw skip_test(); + } + + std::string proc_name("test-output-pgsql"), input_file("-"); + char *argv[] = { &proc_name[0], &input_file[0], NULL }; + + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options = options_t::parse(2, argv); + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + options.style = "default.style"; + options.flat_node_cache_enabled = true; + options.flat_node_file = boost::optional("tests/test_output_pgsql_area_way.flat.nodes.bin"); + + boost::shared_ptr out_test(new output_pgsql_t(mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("libxml2", "tests/test_output_pgsql_way_area.osm", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/test_output_pgsql_way_area.osm'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + assert_has_table(test_conn, "osm2pgsql_test_point"); + assert_has_table(test_conn, "osm2pgsql_test_line"); + assert_has_table(test_conn, "osm2pgsql_test_polygon"); + assert_has_table(test_conn, "osm2pgsql_test_roads"); + + check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_point"); + check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_line"); + check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_roads"); + check_count(test_conn, 1, "SELECT count(*) FROM osm2pgsql_test_polygon"); +} + +void test_route_rel() { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + throw skip_test(); + } + + std::string proc_name("test-output-pgsql"), input_file("-"); + char *argv[] = { &proc_name[0], &input_file[0], NULL }; + + boost::shared_ptr mid_ram(new middle_ram_t()); + options_t options = options_t::parse(2, argv); + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 0; + options.style = "default.style"; + + boost::shared_ptr out_test(new output_pgsql_t(mid_ram.get(), options)); + + osmdata_t osmdata(mid_ram, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("libxml2", "tests/test_output_pgsql_route_rel.osm", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/test_output_pgsql_way_area.osm'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + assert_has_table(test_conn, "osm2pgsql_test_point"); + assert_has_table(test_conn, "osm2pgsql_test_line"); + assert_has_table(test_conn, "osm2pgsql_test_polygon"); + assert_has_table(test_conn, "osm2pgsql_test_roads"); + + check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_point"); + check_count(test_conn, 2, "SELECT count(*) FROM osm2pgsql_test_line"); + check_count(test_conn, 1, "SELECT count(*) FROM osm2pgsql_test_roads"); + check_count(test_conn, 0, "SELECT count(*) FROM osm2pgsql_test_polygon"); +} + +// test the same, but clone the output. it should +// behave the same as the original. +void test_clone() { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + throw skip_test(); + } + + std::string proc_name("test-output-pgsql"), input_file("-"); + char *argv[] = { &proc_name[0], &input_file[0], NULL }; + + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options = options_t::parse(2, argv); + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + options.style = "default.style"; + + output_pgsql_t out_test(mid_pgsql.get(), options); + + //TODO: make the middle testable too + //boost::shared_ptr mid_clone = mid_pgsql->get_instance(); + boost::shared_ptr out_clone = out_test.clone(mid_pgsql.get()); + + osmdata_t osmdata(mid_pgsql, out_clone); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + assert_has_table(test_conn, "osm2pgsql_test_point"); + assert_has_table(test_conn, "osm2pgsql_test_line"); + assert_has_table(test_conn, "osm2pgsql_test_polygon"); + assert_has_table(test_conn, "osm2pgsql_test_roads"); + + check_count(test_conn, 1342, "SELECT count(*) FROM osm2pgsql_test_point"); + check_count(test_conn, 3300, "SELECT count(*) FROM osm2pgsql_test_line"); + check_count(test_conn, 375, "SELECT count(*) FROM osm2pgsql_test_roads"); + check_count(test_conn, 4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); +} + +} // anonymous namespace + +int main(int argc, char *argv[]) { + RUN_TEST(test_regression_simple); + RUN_TEST(test_latlong); + RUN_TEST(test_clone); + RUN_TEST(test_area_way_simple); + RUN_TEST(test_route_rel); + + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test_output_pgsql_route_rel.osm osm2pgsql-0.88.1/tests/test_output_pgsql_route_rel.osm --- osm2pgsql-0.82.0/tests/test_output_pgsql_route_rel.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_pgsql_route_rel.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test-output-pgsql-tablespace.cpp osm2pgsql-0.88.1/tests/test-output-pgsql-tablespace.cpp --- osm2pgsql-0.82.0/tests/test-output-pgsql-tablespace.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-pgsql-tablespace.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,146 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "output-pgsql.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "middle-ram.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +namespace { + +struct skip_test : public std::exception { + const char *what() { return "Test skipped."; } +}; + +void run_test(const char* test_name, void (*testfunc)()) { + try { + fprintf(stderr, "%s\n", test_name); + testfunc(); + + } catch (const skip_test &) { + exit(77); // <-- code to skip this test. + + } catch (const std::exception& e) { + fprintf(stderr, "%s\n", e.what()); + fprintf(stderr, "FAIL\n"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "PASS\n"); +} +#define RUN_TEST(x) run_test(#x, &(x)) + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +void assert_has_table(pg::conn_ptr &test_conn, const std::string &table_name) { + std::string query = (boost::format("select count(*) from pg_catalog.pg_class " + "where relname = '%1%'") + % table_name).str(); + + check_count(test_conn, 1, query); +} + +// "simple" test modeled on the basic regression test from +// the python script. this is just to check everything is +// working as expected before we start the complex stuff. +void test_regression_simple() { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + db->check_tblspc(); // Unlike others, these tests require a test tablespace + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + throw skip_test(); + } + + std::string proc_name("test-output-pgsql"), input_file("-"); + char *argv[] = { &proc_name[0], &input_file[0], NULL }; + + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options = options_t::parse(2, argv); + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.slim = 1; + options.style = "default.style"; + + options.tblsslim_index = "tablespacetest"; + options.tblsslim_data = "tablespacetest"; + + boost::shared_ptr out_test(new output_pgsql_t(mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("pbf", "tests/liechtenstein-2013-08-03.osm.pbf", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/liechtenstein-2013-08-03.osm.pbf'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + assert_has_table(test_conn, "osm2pgsql_test_point"); + assert_has_table(test_conn, "osm2pgsql_test_line"); + assert_has_table(test_conn, "osm2pgsql_test_polygon"); + assert_has_table(test_conn, "osm2pgsql_test_roads"); + + check_count(test_conn, 1342, "SELECT count(*) FROM osm2pgsql_test_point"); + check_count(test_conn, 3300, "SELECT count(*) FROM osm2pgsql_test_line"); + check_count(test_conn, 375, "SELECT count(*) FROM osm2pgsql_test_roads"); + check_count(test_conn, 4128, "SELECT count(*) FROM osm2pgsql_test_polygon"); +} + +} // anonymous namespace + +int main(int argc, char *argv[]) { + RUN_TEST(test_regression_simple); + + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test_output_pgsql_way_area.osm osm2pgsql-0.88.1/tests/test_output_pgsql_way_area.osm --- osm2pgsql-0.82.0/tests/test_output_pgsql_way_area.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_pgsql_way_area.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test-output-pgsql-z_order.cpp osm2pgsql-0.88.1/tests/test-output-pgsql-z_order.cpp --- osm2pgsql-0.82.0/tests/test-output-pgsql-z_order.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-output-pgsql-z_order.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,186 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "output-pgsql.hpp" +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "middle-ram.hpp" +#include "taginfo_impl.hpp" +#include "parse.hpp" + +#include +#include +#include + +#include +#include + +#include "tests/middle-tests.hpp" +#include "tests/common-pg.hpp" + +namespace { + +struct skip_test : public std::exception { + const char *what() { return "Test skipped."; } +}; + +void run_test(const char* test_name, void (*testfunc)()) { + try { + fprintf(stderr, "%s\n", test_name); + testfunc(); + + } catch (const skip_test &) { + exit(77); // <-- code to skip this test. + + } catch (const std::exception& e) { + fprintf(stderr, "%s\n", e.what()); + fprintf(stderr, "FAIL\n"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "PASS\n"); +} +#define RUN_TEST(x) run_test(#x, &(x)) + +void check_string(pg::conn_ptr &conn, std::string expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check a string, but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string actual = PQgetvalue(res->get(), 0, 0); + + if (actual != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % actual % query).str()); + } +} + + +void check_count(pg::conn_ptr &conn, int expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query " + "to check COUNT(*), but got %1%. Query " + "was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + int count = boost::lexical_cast(numstr); + + if (count != expected) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % count % query).str()); + } +} + +void check_number(pg::conn_ptr &conn, double expected, const std::string &query) { + pg::result_ptr res = conn->exec(query); + + int ntuples = PQntuples(res->get()); + if (ntuples != 1) { + throw std::runtime_error((boost::format("Expected only one tuple from a query, " + " but got %1%. Query was: %2%.") + % ntuples % query).str()); + } + + std::string numstr = PQgetvalue(res->get(), 0, 0); + double num = boost::lexical_cast(numstr); + + // floating point isn't exact, so allow a 0.01% difference + if ((num > 1.0001*expected) || (num < 0.9999*expected)) { + throw std::runtime_error((boost::format("Expected %1%, but got %2%, when running " + "query: %3%.") + % expected % num % query).str()); + } +} + +void assert_has_table(pg::conn_ptr &test_conn, const std::string &table_name) { + std::string query = (boost::format("select count(*) from pg_catalog.pg_class " + "where relname = '%1%'") + % table_name).str(); + + check_count(test_conn, 1, query); +} + +// "simple" test modeled on the basic regression test from +// the python script. this is just to check everything is +// working as expected before we start the complex stuff. +void test_z_order() { + boost::scoped_ptr db; + + try { + db.reset(new pg::tempdb); + } catch (const std::exception &e) { + std::cerr << "Unable to setup database: " << e.what() << "\n"; + throw skip_test(); + } + + std::string proc_name("test-output-pgsql"), input_file("-"); + char *argv[] = { &proc_name[0], &input_file[0], NULL }; + + boost::shared_ptr mid_pgsql(new middle_pgsql_t()); + options_t options = options_t::parse(2, argv); + options.conninfo = db->conninfo().c_str(); + options.num_procs = 1; + options.prefix = "osm2pgsql_test"; + options.style = "default.style"; + + boost::shared_ptr out_test(new output_pgsql_t(mid_pgsql.get(), options)); + + osmdata_t osmdata(mid_pgsql, out_test); + + boost::scoped_ptr parser(new parse_delegate_t(options.extra_attributes, options.bbox, options.projection)); + + osmdata.start(); + + if (parser->streamFile("libxml2", "tests/test_output_pgsql_z_order.osm", options.sanitize, &osmdata) != 0) { + throw std::runtime_error("Unable to read input file `tests/test_output_pgsql_z_order.osm'."); + } + + parser.reset(NULL); + + osmdata.stop(); + + // start a new connection to run tests on + pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); + + assert_has_table(test_conn, "osm2pgsql_test_point"); + assert_has_table(test_conn, "osm2pgsql_test_line"); + assert_has_table(test_conn, "osm2pgsql_test_polygon"); + assert_has_table(test_conn, "osm2pgsql_test_roads"); + + check_string(test_conn, "motorway", "SELECT highway FROM osm2pgsql_test_line WHERE layer IS NULL ORDER BY z_order DESC LIMIT 1 OFFSET 0"); + check_string(test_conn, "trunk", "SELECT highway FROM osm2pgsql_test_line WHERE layer IS NULL ORDER BY z_order DESC LIMIT 1 OFFSET 1"); + check_string(test_conn, "primary", "SELECT highway FROM osm2pgsql_test_line WHERE layer IS NULL ORDER BY z_order DESC LIMIT 1 OFFSET 2"); + check_string(test_conn, "secondary", "SELECT highway FROM osm2pgsql_test_line WHERE layer IS NULL ORDER BY z_order DESC LIMIT 1 OFFSET 3"); + check_string(test_conn, "tertiary", "SELECT highway FROM osm2pgsql_test_line WHERE layer IS NULL ORDER BY z_order DESC LIMIT 1 OFFSET 4"); + + check_string(test_conn, "residential", "SELECT highway FROM osm2pgsql_test_line ORDER BY z_order DESC LIMIT 1 OFFSET 0"); +} + +} // anonymous namespace + +int main(int argc, char *argv[]) { + RUN_TEST(test_z_order); + + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test_output_pgsql_z_order.osm osm2pgsql-0.88.1/tests/test_output_pgsql_z_order.osm --- osm2pgsql-0.82.0/tests/test_output_pgsql_z_order.osm 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test_output_pgsql_z_order.osm 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff -Nru osm2pgsql-0.82.0/tests/test-parse-options.cpp osm2pgsql-0.88.1/tests/test-parse-options.cpp --- osm2pgsql-0.82.0/tests/test-parse-options.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-parse-options.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,296 @@ +#include "options.hpp" +#include "middle-pgsql.hpp" +#include "middle-ram.hpp" +#include "output-pgsql.hpp" +#include "output-gazetteer.hpp" +#include "output-null.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace alg = boost::algorithm; + +#define len(x) sizeof(x)/sizeof(x[0]) + +void run_test(const char* test_name, void (*testfunc)()) +{ + try + { + fprintf(stderr, "%s\n", test_name); + testfunc(); + } + catch(const std::exception& e) + { + fprintf(stderr, "%s\n", e.what()); + fprintf(stderr, "FAIL\n"); + exit(EXIT_FAILURE); + } + fprintf(stderr, "PASS\n"); +} + +void parse_fail(const int argc, const char* argv[], const std::string& fail_message) +{ + try + { + options_t options = options_t::parse(argc, const_cast(argv)); + throw std::logic_error((boost::format("Expected '%1%'") % fail_message).str()); + } + catch(const std::runtime_error& e) + { + if(!alg::icontains(e.what(), fail_message)) + throw std::logic_error((boost::format("Expected '%1%' but instead got '%2%'") % fail_message % e.what()).str()); + } +} + +void test_insufficient_args() +{ + const char* argv[] = {"osm2pgsql", "-a", "-c", "--slim"}; + parse_fail(len(argv), argv, "usage error"); +} + +void test_incompatible_args() +{ + const char* a1[] = {"osm2pgsql", "-a", "-c", "--slim", "tests/liechtenstein-2013-08-03.osm.pbf"}; + parse_fail(len(a1), a1, "options can not be used at the same time"); + + const char* a2[] = {"osm2pgsql", "--drop", "tests/liechtenstein-2013-08-03.osm.pbf"}; + parse_fail(len(a2), a2, "drop only makes sense with"); + + const char* a3[] = {"osm2pgsql", "-j", "-k", "tests/liechtenstein-2013-08-03.osm.pbf"}; + parse_fail(len(a3), a3, "you can not specify both"); +} + +void test_middles() +{ + const char* a1[] = {"osm2pgsql", "--slim", "tests/liechtenstein-2013-08-03.osm.pbf"}; + options_t options = options_t::parse(len(a1), const_cast(a1)); + boost::shared_ptr mid = middle_t::create_middle(options.slim); + if(dynamic_cast(mid.get()) == NULL) + { + throw std::logic_error("Using slim mode we expected a pgsql middle"); + } + + const char* a2[] = {"osm2pgsql", "tests/liechtenstein-2013-08-03.osm.pbf"}; + options = options_t::parse(len(a2), const_cast(a2)); + mid = middle_t::create_middle(options.slim); + if(dynamic_cast(mid.get()) == NULL) + { + throw std::logic_error("Using without slim mode we expected a ram middle"); + } +} + +void test_outputs() +{ + const char* a1[] = {"osm2pgsql", "-O", "pgsql", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"}; + options_t options = options_t::parse(len(a1), const_cast(a1)); + boost::shared_ptr mid = middle_t::create_middle(options.slim); + std::vector > outs = output_t::create_outputs(mid.get(), options); + output_t* out = outs.front().get(); + if(dynamic_cast(out) == NULL) + { + throw std::logic_error("Expected a pgsql output"); + } + + const char* a2[] = {"osm2pgsql", "-O", "gazetteer", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"}; + options = options_t::parse(len(a2), const_cast(a2)); + mid = middle_t::create_middle(options.slim); + outs = output_t::create_outputs(mid.get(), options); + out = outs.front().get(); + if(dynamic_cast(out) == NULL) + { + throw std::logic_error("Expected a gazetteer output"); + } + + const char* a3[] = {"osm2pgsql", "-O", "null", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"}; + options = options_t::parse(len(a3), const_cast(a3)); + mid = middle_t::create_middle(options.slim); + outs = output_t::create_outputs(mid.get(), options); + out = outs.front().get(); + if(dynamic_cast(out) == NULL) + { + throw std::logic_error("Expected a null output"); + } + + const char* a4[] = {"osm2pgsql", "-O", "keine_richtige_ausgabe", "--style", "default.style", "tests/liechtenstein-2013-08-03.osm.pbf"}; + options = options_t::parse(len(a4), const_cast(a4)); + mid = middle_t::create_middle(options.slim); + try + { + outs = output_t::create_outputs(mid.get(), options); + out = outs.front().get(); + throw std::logic_error("Expected 'not recognised'"); + } + catch(const std::runtime_error& e) + { + if(!alg::icontains(e.what(), "not recognised")) + throw std::logic_error((boost::format("Expected 'not recognised' but instead got '%2%'") % e.what()).str()); + } +} + +int get_random_proj(std::vector& args) +{ + int proj = rand() % (PROJ_COUNT + 1); + switch(proj) + { + case PROJ_LATLONG: + case PROJ_SPHERE_MERC: + args.push_back(reprojection(proj).project_getprojinfo()->option); + break; + default: + args.push_back("--proj"); + //nice contiguous block of valid epsgs here randomly use one of those.. + proj = (rand() % (2962 - 2308)) + 2308; + args.push_back((boost::format("%1%") % proj).str()); + proj = -proj; + break; + } + return proj; +} + +std::string get_random_string(const int length) +{ + std::string charset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"); + std::string result; + result.resize(length); + + for (int i = 0; i < length; i++) + result[i] = charset[rand() % charset.length()]; + + return result; +} + +template +void add_arg_or_not(const char* arg, std::vector& args, T& option) +{ + if(rand() % 2) + { + args.push_back(arg); + option = 1; + } + else + option = 0; +} + +void add_arg_and_val_or_not(const char* arg, std::vector& args, int option, const int val) +{ + if(rand() % 2) + { + args.push_back(arg); + args.push_back((boost::format("%1%") % val).str()); + option = val; + } +} + +void add_arg_and_val_or_not(const char* arg, std::vector& args, const char *option, std::string val) +{ + if(rand() % 2) + { + args.push_back(arg); + args.push_back(val); + option = val.c_str(); + } +} + +void test_random_perms() +{ + + for(int i = 0; i < 5; ++i) + { + options_t options; + std::vector args; + args.push_back("osm2pgsql"); + + //pick a projection + options.projection.reset(new reprojection(get_random_proj(args))); + + //pick a style file + std::string style = get_random_string(15); + options.style = style.c_str(); + args.push_back("--style"); + args.push_back(style); + + add_arg_and_val_or_not("--cache", args, options.cache, rand() % 800); + add_arg_and_val_or_not("--database", args, options.db.c_str(), get_random_string(6)); + if (options.username) { + add_arg_and_val_or_not("--username", args, options.username->c_str(), get_random_string(6)); + } + if (options.host) { + add_arg_and_val_or_not("--host", args, options.host->c_str(), get_random_string(6)); + } + //add_arg_and_val_or_not("--port", args, options.port, rand() % 9999); + + //--hstore-match-only + //--hstore-column Add an additional hstore (key/value) column containing all tags that start with the specified string, eg --hstore-column "name:" will produce an extra hstore column that contains all name:xx tags + + add_arg_or_not("--hstore-add-index", args, options.enable_hstore_index); + add_arg_or_not("--utf8-sanitize", args, options.sanitize); + + //--tablespace-index The name of the PostgreSQL tablespace where all indexes will be created. The following options allow more fine-grained control: + // --tablespace-main-data tablespace for main tables + // --tablespace-main-index tablespace for main table indexes + // --tablespace-slim-data tablespace for slim mode tables + // --tablespace-slim-index tablespace for slim mode indexes + // (if unset, use db's default; -i is equivalent to setting + // --tablespace-main-index and --tablespace-slim-index) + + add_arg_and_val_or_not("--number-processes", args, options.num_procs, rand() % 12); + + //add_arg_or_not("--disable-parallel-indexing", args, options.parallel_indexing); + + add_arg_or_not("--unlogged", args, options.unlogged); + + //--cache-strategy Specifies the method used to cache nodes in ram. Available options are: dense chunk sparse optimized + + if (options.flat_node_file) { + add_arg_and_val_or_not("--flat-nodes", args, options.flat_node_file->c_str(), get_random_string(15)); + } + + //--expire-tiles [min_zoom-]max_zoom Create a tile expiry list. + + add_arg_and_val_or_not("--expire-output", args, options.expire_tiles_filename.c_str(), get_random_string(15)); + + //--bbox Apply a bounding box filter on the imported data Must be specified as: minlon,minlat,maxlon,maxlat e.g. --bbox -0.5,51.25,0.5,51.75 + + add_arg_and_val_or_not("--prefix", args, options.prefix.c_str(), get_random_string(15)); + + //--input-reader Input frontend. libxml2 - Parse XML using libxml2. (default) primitive - Primitive XML parsing. pbf - OSM binary format. + + if (options.tag_transform_script) { + add_arg_and_val_or_not("--tag-transform-script", args, options.tag_transform_script->c_str(), get_random_string(15)); + } + add_arg_or_not("--extra-attributes", args, options.extra_attributes); + add_arg_or_not("--multi-geometry", args, options.enable_multi); + add_arg_or_not("--keep-coastlines", args, options.keep_coastlines); + add_arg_or_not("--exclude-invalid-polygon", args, options.excludepoly); + + //add the input file + args.push_back("tests/liechtenstein-2013-08-03.osm.pbf"); + + const char** argv = new const char*[args.size() + 1]; + argv[args.size()] = NULL; + for(std::vector::const_iterator arg = args.begin(); arg != args.end(); ++arg) + argv[arg - args.begin()] = arg->c_str(); + options_t::parse(args.size(), const_cast(argv)); + delete[] argv; + } +} + +int main(int argc, char *argv[]) +{ + srand(0); + + //try each test if any fail we will exit + run_test("test_insufficient_args", test_insufficient_args); + run_test("test_incompatible_args", test_incompatible_args); + run_test("test_middles", test_middles); + run_test("test_outputs", test_outputs); + run_test("test_random_perms", test_random_perms); + + //passed + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test-parse-xml2.cpp osm2pgsql-0.88.1/tests/test-parse-xml2.cpp --- osm2pgsql-0.82.0/tests/test-parse-xml2.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-parse-xml2.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,163 @@ +#include +#include +#include +#include +#include + +#include + +#include "osmtypes.hpp" +#include "osmdata.hpp" +#include "parse-xml2.hpp" +#include "output.hpp" +#include "options.hpp" + +void exit_nicely() +{ + fprintf(stderr, "Error occurred, cleaning up\n"); + exit(1); +} + +struct test_middle_t : public middle_t { + virtual ~test_middle_t() {} + + int start(const options_t *out_options_) { return 0; } + void stop(void) { } + void cleanup(void) { } + void analyze(void) { } + void end(void) { } + void commit(void) { } + + int nodes_set(osmid_t id, double lat, double lon, const taglist_t &tags) { return 0; } + int nodes_get_list(nodelist_t &out, const idlist_t nds) const { return 0; } + + int ways_set(osmid_t id, const idlist_t &nds, const taglist_t &tags) { return 0; } + int ways_get(osmid_t id, taglist_t &tags, nodelist_t &nodes) const { return 0; } + int ways_get_list(const idlist_t &ids, idlist_t &way_ids, + std::vector &tags, + std::vector &nodes) const { return 0; } + + int relations_set(osmid_t id, const memberlist_t &members, const taglist_t &tags) { return 0; } + int relations_get(osmid_t id, memberlist_t &members, taglist_t &tags) const { return 0; } + + void iterate_ways(pending_processor& pf) { } + void iterate_relations(pending_processor& pf) { } + + virtual size_t pending_count() const { return 0; } + + std::vector relations_using_way(osmid_t way_id) const { return std::vector(); } + + virtual boost::shared_ptr get_instance() const {return boost::shared_ptr();} +}; + +struct test_output_t : public output_t { + uint64_t sum_ids, num_nodes, num_ways, num_relations, num_nds, num_members; + + explicit test_output_t(const options_t &options_) + : output_t(NULL, options_), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0), + num_nds(0), num_members(0) { + } + + explicit test_output_t(const test_output_t &other) + : output_t(this->m_mid, this->m_options), sum_ids(0), num_nodes(0), num_ways(0), num_relations(0), + num_nds(0), num_members(0) { + } + + virtual ~test_output_t() { + } + + boost::shared_ptr clone(const middle_query_t *cloned_middle) const{ + test_output_t *clone = new test_output_t(*this); + clone->m_mid = cloned_middle; + return boost::shared_ptr(clone); + } + + int node_add(osmid_t id, double lat, double lon, const taglist_t &tags) { + assert(id > 0); + sum_ids += id; + num_nodes += 1; + return 0; + } + + int way_add(osmid_t id, const idlist_t &nds, const taglist_t &tags) { + assert(id > 0); + sum_ids += id; + num_ways += 1; + assert(nds.size() >= 0); + num_nds += uint64_t(nds.size()); + return 0; + } + + int relation_add(osmid_t id, const memberlist_t &members, const taglist_t &tags) { + assert(id > 0); + sum_ids += id; + num_relations += 1; + assert(members.size() >= 0); + num_members += uint64_t(members.size()); + return 0; + } + + int start() { return 0; } + int connect(int startTransaction) { return 0; } + void stop() { } + void commit() { } + void cleanup(void) { } + void close(int stopTransaction) { } + + void enqueue_ways(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { } + int pending_way(osmid_t id, int exists) { return 0; } + + void enqueue_relations(pending_queue_t &job_queue, osmid_t id, size_t output_id, size_t& added) { } + int pending_relation(osmid_t id, int exists) { return 0; } + + int node_modify(osmid_t id, double lat, double lon, const taglist_t &tags) { return 0; } + int way_modify(osmid_t id, const idlist_t &nds, const taglist_t &tags) { return 0; } + int relation_modify(osmid_t id, const memberlist_t &members, const taglist_t &tags) { return 0; } + + int node_delete(osmid_t id) { return 0; } + int way_delete(osmid_t id) { return 0; } + int relation_delete(osmid_t id) { return 0; } + +}; + + +void assert_equal(uint64_t actual, uint64_t expected) { + if (actual != expected) { + std::cerr << "Expected " << expected << ", but got " << actual << ".\n"; + exit(1); + } +} + +int main(int argc, char *argv[]) { + char *srcdir = getenv("srcdir"); + + if (srcdir == NULL) { + std::cerr << "$srcdir not set!\n"; + return 1; + } + + std::string inputfile = std::string(srcdir) + std::string("/tests/test_multipolygon.osm"); + + options_t options; + boost::shared_ptr projection(new reprojection(PROJ_SPHERE_MERC)); + options.projection = projection; + + boost::shared_ptr out_test(new test_output_t(options)); + osmdata_t osmdata(boost::make_shared(), out_test); + + parse_xml2_t parser(0, false, projection, 0, 0, 0, 0); + + int ret = parser.streamFile(inputfile.c_str(), 0, &osmdata); + if (ret != 0) { + return ret; + } + + assert_equal(out_test->sum_ids, 73514L); + assert_equal(out_test->num_nodes, 353L); + assert_equal(out_test->num_ways, 140L); + assert_equal(out_test->num_relations, 40L); + assert_equal(out_test->num_nds, 495L); + assert_equal(out_test->num_members, 146L); + + return 0; +} diff -Nru osm2pgsql-0.82.0/tests/test-pgsql-escape.cpp osm2pgsql-0.88.1/tests/test-pgsql-escape.cpp --- osm2pgsql-0.82.0/tests/test-pgsql-escape.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/tests/test-pgsql-escape.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,22 @@ +#include +#include "pgsql.hpp" + +void test_escape(const char *in, const char *out) { + std::string sql; + escape(in, sql); + if (sql.compare(out) != 0) { + std::cerr << "Expected " << out << ", but got " << sql << " for " << in <<".\n"; + exit(1); + } +} + +int main(int argc, char *argv[]) { + std::string sql; + test_escape("farmland", "farmland"); + test_escape("", ""); + test_escape("\\", "\\\\"); + test_escape("foo\nbar", "foo\\\nbar"); + test_escape("\t\r\n", "\\\t\\\r\\\n"); + + return 0; +} diff -Nru osm2pgsql-0.82.0/text-tree.c osm2pgsql-0.88.1/text-tree.c --- osm2pgsql-0.82.0/text-tree.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/text-tree.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,111 +0,0 @@ -/* text-tree.c - * - * Storage of reference counted text strings - * used by keyvals.c to store the key/value strings - */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include "text-tree.h" - -struct tree_context *tree_ctx = NULL; - -int text_compare(const void *pa, const void *pb, void *rb_param) -{ - struct text_node *a = (struct text_node *)pa; - struct text_node *b = (struct text_node *)pb; - - rb_param = NULL; - return strcmp(a->str, b->str); -} - -struct tree_context *text_init(void) -{ - struct tree_context *context; - struct rb_table *table = rb_create (text_compare, NULL, NULL); - - assert(table); - context = calloc(1, sizeof(*context)); - assert(context); - context->table = table; - tree_ctx = context; - return context; -} - -void text_free(void *pa, void *rb_param) -{ - struct text_node *a = (struct text_node *)pa; - rb_param = NULL; - free(a->str); - free(a); -} - -const char *text_get(struct tree_context *context, const char *text) -{ - struct text_node *node, *dupe; - - node = malloc(sizeof(*node)); - assert(node); - - node->str = strdup(text); - assert(node->str); - node->ref = 0; - dupe = rb_insert(context->table, (void *)node); - if (dupe) { - free(node->str); - free(node); - dupe->ref++; - return dupe->str; - } else { - node->ref++; - return node->str; - } -} - - -void text_release(struct tree_context *context, const char *text) -{ - struct text_node *node, find; - - find.str = (char *)text; - find.ref = 0; - node = rb_find(context->table, (void *)&find); - if (!node) { - fprintf(stderr, "failed to find '%s'\n", text); - return; - } - node->ref--; - if (!node->ref) { - rb_delete (context->table, &find); - free(node->str); - free(node); - } -} - -void text_exit(void) -{ - struct tree_context *context = tree_ctx; - rb_destroy(context->table, text_free); - free(context); - tree_ctx = NULL; -} -#if 0 -int main(int argc, char **argv) -{ - struct tree_context *ctx = text_init(); - - printf("%1$p %1$s\n", text_get(ctx, "Hello")); - printf("%1$p %1$s\n", text_get(ctx, "Hello")); - printf("%1$p %1$s\n", text_get(ctx, "World")); - - text_release(ctx,"Hello"); - text_release(ctx,"Hello"); - text_release(ctx,"World"); - text_release(ctx,"Hello"); - - text_exit(ctx); - return 0; -} -#endif diff -Nru osm2pgsql-0.82.0/text-tree.h osm2pgsql-0.88.1/text-tree.h --- osm2pgsql-0.82.0/text-tree.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/text-tree.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,21 +0,0 @@ -#ifndef TEXT_TREE_H -#define TEXT_TREE_H - -#include "rb.h" - -struct tree_context { - struct rb_table *table; -}; - -extern struct tree_context *tree_ctx; - -struct text_node { - char *str; - int ref; -}; - -struct tree_context *text_init(void); -void text_exit(void); -const char *text_get(struct tree_context *context, const char *text); -void text_release(struct tree_context *context, const char *text); -#endif diff -Nru osm2pgsql-0.82.0/.travis.yml osm2pgsql-0.88.1/.travis.yml --- osm2pgsql-0.82.0/.travis.yml 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/.travis.yml 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,74 @@ +language: cpp + +addons: + apt: + packages: + - libtool + - libxml2-dev + - libgeos-dev + - libgeos++-dev + - libpq-dev + - libbz2-dev + - libproj-dev + - protobuf-c-compiler + - libprotobuf-c0-dev + +matrix: + include: + #basic tests of with/without lua and old boost versions, with both clang and gc + - os: linux + compiler: clang + env: USE_LUA=true + - os: linux + compiler: clang + env: USE_LUA=true BOOST_PPA=1.55 + - os: linux + compiler: clang + env: USE_LUA=false BOOST_PPA=1.55 + - os: linux + compiler: gcc + env: USE_LUA=true + - os: linux + compiler: gcc + env: USE_LUA=true BOOST_PPA=1.55 + - os: linux + compiler: gcc + env: USE_LUA=false BOOST_PPA=1.55 + # additional tests + - os: linux + compiler: clang + env: USE_LUA=true BOOST_PPA=1.54 + +before_install: + # BOOST_PPA is only set on linux + - if [ -z ${BOOST_PPA+x} ]; then + echo "Using system Boost"; + else + echo "Using Boost PPA"; + sudo add-apt-repository -y ppa:boost-latest/ppa; + fi + - if [[ $(uname -s) == 'Linux' ]]; then + sudo apt-get update -qq; + fi +install: + - if [[ $(uname -s) == 'Linux' ]]; then + if [ -z ${BOOST_PPA+x} ]; then + sudo apt-get install -y -qq libboost1.48-all-dev; + else + sudo apt-get install -y -qq "boost${BOOST_PPA}"; + fi; + if [[ "${USE_LUA}" = "true" ]]; then + sudo apt-get install -y -qq lua5.2 liblua5.2-dev; + fi; + fi; +before_script: + - xml2-config --version + - geos-config --version + - proj | head -n1 + - if [ "${USE_LUA}" = "true" ]; then + lua -v; + fi; +script: + ./autogen.sh && ./configure && make -j2 +after_failure: + - cat config.log diff -Nru osm2pgsql-0.82.0/UTF8sanitizer.c osm2pgsql-0.88.1/UTF8sanitizer.c --- osm2pgsql-0.82.0/UTF8sanitizer.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/UTF8sanitizer.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,164 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "sanitizer.h" -#include "input.h" - -int sanitizerClose(void *context); -int sanitizerProcess(void *context, char *buffer, int len); - - -/* UTF8sanitizer algorithm has some nasty edge cases when trying to operate - * in a 'block at a time' mode. For example, in the following scenario: - * - * INPUT sequence is 2 buffers with a 6 byte char starting at X1 - * - * [ len = 5 ] [len = 1] - * X1 X2 X3 X4 X5 X6 - * - * OUTPUT: nothing is generated for first buffer - * This will itself cause caller to assume EOF (hopefully normal reader will read >> 5 bytes). - * subsequent read of len=1 whille return all 6 bytes potentially causing output buffer overflow (and overwriting input data) - * - * The solution is to provice a small output buffer to hold anything bigger than a single byte - * - */ - - -struct Context { - long long line; - long long chars1, chars2, chars3, chars4, chars5, chars6; - int state, current_size; - int long_char[6]; - int out_char[10]; - int pend; - int verbose; - void *file; -}; - - -int sanitizerClose(void *context) -{ - struct Context *ctx = context; - int r = inputClose(ctx->file); - - if (ctx->verbose) { - fprintf(stderr, "Summary:\n"); - fprintf(stderr, "chars1: %lld\n", ctx->chars1); - fprintf(stderr, "chars2: %lld\n", ctx->chars2); - fprintf(stderr, "chars3: %lld\n", ctx->chars3); - fprintf(stderr, "chars4: %lld\n", ctx->chars4); - fprintf(stderr, "chars5: %lld\n", ctx->chars5); - fprintf(stderr, "chars6: %lld\n", ctx->chars6); - fprintf(stderr, "lines : %lld\n", ctx->line); - } - - free(ctx); - return r; -} - -xmlTextReaderPtr sanitizerOpen(const char *name) -{ - struct Context *ctx = malloc (sizeof(*ctx)); - - if (!ctx) - return NULL; - - memset(ctx, 0, sizeof(*ctx)); - ctx->verbose = 0; - ctx->state = 1; - ctx->pend = 0; - - ctx->file = inputOpen(name); - if (!ctx->file) { - fprintf(stderr, "Input reader create failed\n"); - free(ctx); - return NULL; - } - - return xmlReaderForIO(sanitizerProcess, sanitizerClose, (void *)ctx, NULL, NULL, 0); -} - - -int sanitizerProcess(void *context, char *buffer, int len) -{ - struct Context *ctx = context; - int current_char, i, out = 0; - - while (out < len) { - if (ctx->pend) { - buffer[out++] = ctx->out_char[--ctx->pend]; - continue; - } - - current_char=inputGetChar(ctx->file); - if (inputEof(ctx->file)) - break; - - if ((current_char & 128) == 0) { - /* Handle_ASCII_char(); */ - if (current_char == '\n') - ctx->line++; - else - ctx->chars1++; - if (ctx->state != 1) { - if (ctx->verbose) - fprintf(stderr, "Error at line %lld\n", ctx->line); - buffer[out++] = '_'; - ctx->state = 1; - } - /* buffer[out++] = current_char; */ - ctx->out_char[ctx->pend++] = current_char; - } else if ((current_char & (128+64)) == 128) { - /* Handle_continue_char(); */ - if(ctx->state > 1) { - ctx->state--; - if(ctx->state==1) { - ctx->out_char[ctx->pend++] = current_char; - for(i=ctx->current_size-1; i>0; i--) { - ctx->out_char[ctx->pend++] = ctx->long_char[i-1]; - } - } - } else { - if (ctx->verbose) - fprintf(stderr, "Error at line %lld\n", ctx->line); - buffer[out++] = '_'; - ctx->state=1; - } - } else if ((current_char & (128+64+32)) == (128+64)) { - /* Handle_two_bytes(); */ - ctx->state=2; - ctx->chars2++; - ctx->current_size=2; - } else if ((current_char & (128+64+32+16)) == (128+64+32)) { - /* Handle_three_bytes(); */ - ctx->state=3; - ctx->chars3++; - ctx->current_size=3; - } else if ((current_char & (128+64+32+16+8)) == (128+64+32+16)) { - /* Handle_four_bytes(); */ - ctx->state=4; - ctx->chars4++; - ctx->current_size=4; - } else if ((current_char & (128+64+32+16+8+4)) == (128+64+32+16+8)) { - /* Handle_five_bytes(); */ - ctx->state=5; - ctx->chars5++; - ctx->current_size=5; - } else if ((current_char & (128+64+32+16+8+4+2)) == (128+64+32+16+8+4)) { - /* Handle_six_bytes(); */ - ctx->state=6; - ctx->chars6++; - ctx->current_size=6; - } - if(ctx->state>1) { - ctx->long_char[ctx->current_size-ctx->state]=current_char; - } - } - return out; -} diff -Nru osm2pgsql-0.82.0/UTF8sanitizer.cpp osm2pgsql-0.88.1/UTF8sanitizer.cpp --- osm2pgsql-0.82.0/UTF8sanitizer.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/UTF8sanitizer.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,160 @@ +#include +#include +#include + +#include "sanitizer.hpp" +#include "input.hpp" + +int sanitizerClose(void *context); +int sanitizerProcess(void *context, char *buffer, int len); + + +/* UTF8sanitizer algorithm has some nasty edge cases when trying to operate + * in a 'block at a time' mode. For example, in the following scenario: + * + * INPUT sequence is 2 buffers with a 6 byte char starting at X1 + * + * [ len = 5 ] [len = 1] + * X1 X2 X3 X4 X5 X6 + * + * OUTPUT: nothing is generated for first buffer + * This will itself cause caller to assume EOF (hopefully normal reader will read >> 5 bytes). + * subsequent read of len=1 whille return all 6 bytes potentially causing output buffer overflow (and overwriting input data) + * + * The solution is to provice a small output buffer to hold anything bigger than a single byte + * + */ + + +struct Context { + long long line; + long long chars1, chars2, chars3, chars4, chars5, chars6; + int state, current_size; + int long_char[6]; + int out_char[10]; + int pend; + int verbose; + Input *file; +}; + + +int sanitizerClose(void *context) +{ + struct Context *ctx = (struct Context *)context; + int r = inputClose(ctx->file); + + if (ctx->verbose) { + fprintf(stderr, "Summary:\n"); + fprintf(stderr, "chars1: %lld\n", ctx->chars1); + fprintf(stderr, "chars2: %lld\n", ctx->chars2); + fprintf(stderr, "chars3: %lld\n", ctx->chars3); + fprintf(stderr, "chars4: %lld\n", ctx->chars4); + fprintf(stderr, "chars5: %lld\n", ctx->chars5); + fprintf(stderr, "chars6: %lld\n", ctx->chars6); + fprintf(stderr, "lines : %lld\n", ctx->line); + } + + free(ctx); + return r; +} + +xmlTextReaderPtr sanitizerOpen(const char *name) +{ + struct Context *ctx = (struct Context *)malloc(sizeof(*ctx)); + + if (!ctx) + return NULL; + + memset(ctx, 0, sizeof(*ctx)); + ctx->verbose = 0; + ctx->state = 1; + ctx->pend = 0; + + ctx->file = inputOpen(name); + if (!ctx->file) { + fprintf(stderr, "Input reader create failed\n"); + free(ctx); + return NULL; + } + + return xmlReaderForIO(sanitizerProcess, sanitizerClose, (void *)ctx, NULL, NULL, 0); +} + + +int sanitizerProcess(void *context, char *buffer, int len) +{ + struct Context *ctx = (struct Context *)context; + int current_char, i, out = 0; + + while (out < len) { + if (ctx->pend) { + buffer[out++] = ctx->out_char[--ctx->pend]; + continue; + } + + current_char=inputGetChar(ctx->file); + if (inputEof(ctx->file)) + break; + + if ((current_char & 128) == 0) { + /* Handle_ASCII_char(); */ + if (current_char == '\n') + ctx->line++; + else + ctx->chars1++; + if (ctx->state != 1) { + if (ctx->verbose) + fprintf(stderr, "Error at line %lld\n", ctx->line); + buffer[out++] = '_'; + ctx->state = 1; + } + /* buffer[out++] = current_char; */ + ctx->out_char[ctx->pend++] = current_char; + } else if ((current_char & (128+64)) == 128) { + /* Handle_continue_char(); */ + if(ctx->state > 1) { + ctx->state--; + if(ctx->state==1) { + ctx->out_char[ctx->pend++] = current_char; + for(i=ctx->current_size-1; i>0; i--) { + ctx->out_char[ctx->pend++] = ctx->long_char[i-1]; + } + } + } else { + if (ctx->verbose) + fprintf(stderr, "Error at line %lld\n", ctx->line); + buffer[out++] = '_'; + ctx->state=1; + } + } else if ((current_char & (128+64+32)) == (128+64)) { + /* Handle_two_bytes(); */ + ctx->state=2; + ctx->chars2++; + ctx->current_size=2; + } else if ((current_char & (128+64+32+16)) == (128+64+32)) { + /* Handle_three_bytes(); */ + ctx->state=3; + ctx->chars3++; + ctx->current_size=3; + } else if ((current_char & (128+64+32+16+8)) == (128+64+32+16)) { + /* Handle_four_bytes(); */ + ctx->state=4; + ctx->chars4++; + ctx->current_size=4; + } else if ((current_char & (128+64+32+16+8+4)) == (128+64+32+16+8)) { + /* Handle_five_bytes(); */ + ctx->state=5; + ctx->chars5++; + ctx->current_size=5; + } else if ((current_char & (128+64+32+16+8+4+2)) == (128+64+32+16+8+4)) { + /* Handle_six_bytes(); */ + ctx->state=6; + ctx->chars6++; + ctx->current_size=6; + } + if(ctx->state>1) { + ctx->long_char[ctx->current_size-ctx->state]=current_char; + } + } + return out; +} diff -Nru osm2pgsql-0.82.0/util.cpp osm2pgsql-0.88.1/util.cpp --- osm2pgsql-0.82.0/util.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/util.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,13 @@ +#include "util.hpp" + +#include +#include + +namespace util { + + void exit_nicely() { + std::cerr << "Error occurred, cleaning up\n"; + exit(EXIT_FAILURE); + } + +} diff -Nru osm2pgsql-0.82.0/util.hpp osm2pgsql-0.88.1/util.hpp --- osm2pgsql-0.82.0/util.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/util.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,10 @@ +#ifndef UTIL_H +#define UTIL_H + +namespace util { + + void exit_nicely(); +} + + +#endif diff -Nru osm2pgsql-0.82.0/wildcmp.c osm2pgsql-0.88.1/wildcmp.c --- osm2pgsql-0.82.0/wildcmp.c 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/wildcmp.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,106 +0,0 @@ -/* Wildcard matching. - - heavily based on wildcmp.c copyright 2002 Jim Kent - -*/ -#include -#include "wildcmp.h" - -static int subMatch(char *str, char *wild) -/* Returns number of characters that match between str and wild up - * to the next wildcard in wild (or up to end of string.). */ -{ -int len = 0; - -for(;;) - { - if(toupper(*str++) != toupper(*wild++) ) - return(0); - ++len; - switch(*wild) - { - case 0: - case '?': - case '*': - return(len); - } - } -} - -int wildMatch(char *wildCard, char *string) -/* does a case sensitive wild card match with a string. - * * matches any string or no character. - * ? matches any single character. - * anything else etc must match the character exactly. - -returns NO_MATCH, FULL_MATCH or WC_MATCH defined in wildcmp.h - -*/ -{ -int matchStar = 0; -int starMatchSize; -int wildmatch=0; - -for(;;) - { -NEXT_WILD: - switch(*wildCard) - { - case 0: /* end of wildcard */ - { - if(matchStar) - { - while(*string++) - ; - return wildmatch ? WC_MATCH : FULL_MATCH; - } - else if(*string) - return NO_MATCH; - else { - return wildmatch ? WC_MATCH : FULL_MATCH; - } - } - case '*': - wildmatch = 1; - matchStar = 1; - break; - case '?': /* anything will do */ - wildmatch = 1; - { - if(*string == 0) - return NO_MATCH; /* out of string, no match for ? */ - ++string; - break; - } - default: - { - if(matchStar) - { - for(;;) - { - if(*string == 0) /* if out of string no match */ - return NO_MATCH; - - /* note matchStar is re-used here for substring - * after star match length */ - if((starMatchSize = subMatch(string,wildCard)) != 0) - { - string += starMatchSize; - wildCard += starMatchSize; - matchStar = 0; - goto NEXT_WILD; - } - ++string; - } - } - - /* default: they must be equal or no match */ - if(toupper(*string) != toupper(*wildCard)) - return NO_MATCH; - ++string; - break; - } - } - ++wildCard; - } -} diff -Nru osm2pgsql-0.82.0/wildcmp.cpp osm2pgsql-0.88.1/wildcmp.cpp --- osm2pgsql-0.82.0/wildcmp.cpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/wildcmp.cpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,106 @@ +/* Wildcard matching. + + heavily based on wildcmp.c copyright 2002 Jim Kent + +*/ +#include +#include "wildcmp.hpp" + +static int subMatch(const char *str, const char *wild) +/* Returns number of characters that match between str and wild up + * to the next wildcard in wild (or up to end of string.). */ +{ +int len = 0; + +for(;;) + { + if(toupper(*str++) != toupper(*wild++) ) + return(0); + ++len; + switch(*wild) + { + case 0: + case '?': + case '*': + return(len); + } + } +} + +int wildMatch(const char *wildCard, const char *string) +/* does a case sensitive wild card match with a string. + * * matches any string or no character. + * ? matches any single character. + * anything else etc must match the character exactly. + +returns NO_MATCH, FULL_MATCH or WC_MATCH defined in wildcmp.h + +*/ +{ +int matchStar = 0; +int starMatchSize; +int wildmatch=0; + +for(;;) + { +NEXT_WILD: + switch(*wildCard) + { + case 0: /* end of wildcard */ + { + if(matchStar) + { + while(*string++) + ; + return wildmatch ? WC_MATCH : FULL_MATCH; + } + else if(*string) + return NO_MATCH; + else { + return wildmatch ? WC_MATCH : FULL_MATCH; + } + } + case '*': + wildmatch = 1; + matchStar = 1; + break; + case '?': /* anything will do */ + wildmatch = 1; + { + if(*string == 0) + return NO_MATCH; /* out of string, no match for ? */ + ++string; + break; + } + default: + { + if(matchStar) + { + for(;;) + { + if(*string == 0) /* if out of string no match */ + return NO_MATCH; + + /* note matchStar is re-used here for substring + * after star match length */ + if((starMatchSize = subMatch(string,wildCard)) != 0) + { + string += starMatchSize; + wildCard += starMatchSize; + matchStar = 0; + goto NEXT_WILD; + } + ++string; + } + } + + /* default: they must be equal or no match */ + if(toupper(*string) != toupper(*wildCard)) + return NO_MATCH; + ++string; + break; + } + } + ++wildCard; + } +} diff -Nru osm2pgsql-0.82.0/wildcmp.h osm2pgsql-0.88.1/wildcmp.h --- osm2pgsql-0.82.0/wildcmp.h 2013-04-15 07:06:03.000000000 +0000 +++ osm2pgsql-0.88.1/wildcmp.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -#define NO_MATCH 0 -#define FULL_MATCH 1 -#define WC_MATCH 2 - -int wildMatch(char *wildCard, char *string); diff -Nru osm2pgsql-0.82.0/wildcmp.hpp osm2pgsql-0.88.1/wildcmp.hpp --- osm2pgsql-0.82.0/wildcmp.hpp 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/wildcmp.hpp 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,10 @@ +#ifndef WILDCMP_H +#define WILDCMP_H + +#define NO_MATCH 0 +#define FULL_MATCH 1 +#define WC_MATCH 2 + +int wildMatch(const char *wildCard, const char *string); + +#endif diff -Nru osm2pgsql-0.82.0/win_fsync.h osm2pgsql-0.88.1/win_fsync.h --- osm2pgsql-0.82.0/win_fsync.h 1970-01-01 00:00:00.000000000 +0000 +++ osm2pgsql-0.88.1/win_fsync.h 2015-08-15 05:28:11.000000000 +0000 @@ -0,0 +1,71 @@ +#ifndef WIN_FSYNC_H +#define WIN_FSYNC_H + +/* Emulate fsync on platforms that lack it, primarily Windows and + cross-compilers like MinGW. + + This is derived from sqlite3 sources. + http://www.sqlite.org/cvstrac/rlog?f=sqlite/src/os_win.c + http://www.sqlite.org/copyright.html + + Written by Richard W.M. Jones + + Copyright (C) 2008-2014 Free Software Foundation, Inc. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with this program. If not, see . */ + +#include +#include +#include + +inline int fsync (int fd) +{ + HANDLE h = (HANDLE) _get_osfhandle (fd); + DWORD err; + + if (h == INVALID_HANDLE_VALUE) + { + errno = EBADF; + return -1; + } + + if (!FlushFileBuffers (h)) + { + /* Translate some Windows errors into rough approximations of Unix + * errors. MSDN is useless as usual - in this case it doesn't + * document the full range of errors. + */ + err = GetLastError (); + switch (err) + { + case ERROR_ACCESS_DENIED: + /* For a read-only handle, fsync should succeed, even though we have + no way to sync the access-time changes. */ + return 0; + + /* eg. Trying to fsync a tty. */ + case ERROR_INVALID_HANDLE: + errno = EINVAL; + break; + + default: + errno = EIO; + } + return -1; + } + + return 0; +} + +#endif \ No newline at end of file