Adjustments here and there.

Version v0.1

process/6_burn_vector_layers.py

@@ -56,6 +56,11 @@ v_rock_outcrops = 3000.0
 v_shelf_ice = 20000.0
 v_ice_sheet = 30000.0
 
+# heading
+print()
+print('============================================================')
+print('                     BURN VECTOR LAYERS                     ')
+
 msg = 'Burning Antarctic ice shelfs in World raster...'
 cmd = f'gdal_rasterize -l {add_shelf_ice_name} -burn {v_shelf_ice} {ADD_SHELF_ICE} {GEBCO_WORLD}'
 run(msg,cmd)
@@ -65,7 +70,7 @@ cmd = f'gdal_rasterize -l {add_ice_sheet_name} -burn {v_ice_sheet} {ADD_ICE_SHEE
 run(msg,cmd)
 
 msg = 'Burning Antarctic rock outcrops in World raster...'
-cmd = f'gdal_rasterize -l {add_rock_outcrop_name} -burn {v_rock_outcrops} {ADD_ROCK_OUTCROP} {GEBCO_WORLD}'
+cmd = f'gdal_rasterize -l {add_rock_outcrop_name} -burn {v_rock_outcrops} -at {ADD_ROCK_OUTCROP} {GEBCO_WORLD}'
 run(msg,cmd)
 
 msg = 'Burning Greenland ice sheet in World raster...'
@@ -93,9 +98,17 @@ cmd = f'gdal_rasterize -l {add_ice_sheet_name} -burn {v_ice_sheet} {ADD_ICE_SHEE
 run(msg,cmd)
 
 msg = 'Burning Antarctic rock outcrops in Antarctic raster...'
-cmd = f'gdal_rasterize -l {add_rock_outcrop_name} -burn {v_rock_outcrops} {ADD_ROCK_OUTCROP} {GEBCO_ANTARCTIC}'
+cmd = f'gdal_rasterize -l {add_rock_outcrop_name} -burn {v_rock_outcrops} -at {ADD_ROCK_OUTCROP} {GEBCO_ANTARCTIC}'
 run(msg,cmd)
 
 msg = 'Burning glaciers in Antarctic raster...'
 cmd = f'gdal_rasterize -l {glims_antarctic_name} -burn {v_ice_sheet} {GLIMS_ANTARCTIC} {GEBCO_ANTARCTIC}'
 run(msg,cmd)
+
+cleanup(ADD_SHELF_ICE)
+cleanup(ADD_ICE_SHEET)
+cleanup(ADD_ROCK_OUTCROP)
+cleanup(GIMP)
+cleanup(GLIMS_WORLD)
+cleanup(GLIMS_ARCTIC)
+cleanup(GLIMS_ANTARCTIC)

process/7_create_basemaps.py

@@ -38,6 +38,17 @@ __status__ = 'Developement'
 from config import *
 
 
+#=============================================================================
+#    GLOBAL VARIABLES
+#=============================================================================
+
+# GDAL creation options for final result
+GDAL_CREATE_OPTIONS_FINAL = '-co TILED=YES -co BLOCKXSIZE=512 -co BLOCKYSIZE=512 -co COPY_SRC_OVERVIEWS=FALSE -co COMPRESS=NONE -co BIGTIFF=IF_NEEDED'
+
+# Method to create overviews
+OVERVIEW_METHOD = 'cubic'
+
+
 #=============================================================================
 #    FUNCTIONS
 #=============================================================================
@@ -56,10 +67,11 @@ def create_basemap(map_params):
     cleanup(TMP_RGB)
     
     msg = f'Finalizing AWI Basemap {name}...'
-    cmd = f'gdal_translate -a_srs {epsg} -co TILED=YES -co BLOCKXSIZE=512 -co BLOCKYSIZE=512 -co COPY_SRC_OVERVIEWS=FALSE -co COMPRESS=NONE -stats -of GTiff {TMP_SHADED} {output}'
+    cmd = f'gdal_translate -a_srs {epsg} -stats -of GTiff {GDAL_CREATE_OPTIONS_FINAL} {TMP_SHADED} {output}'
     run(msg,cmd)
     
     cleanup(TMP_SHADED)
+    cleanup(f'{TMP_SHADED}.aux.xml')
     
     msg = f'Creating pyramids for AWI Basemap {name}...'
     cmd = f'gdaladdo -r {OVERVIEW_METHOD} --config COMPRESS_OVERVIEW NONE --config INTERLEAVE_OVERVIEW PIXEL --config BIGTIFF_OVERVIEW IF_NEEDED --config GDAL_TIFF_OVR_BLOCKSIZE 512 {output} 2 4 8 16 32 64'
@@ -70,6 +82,11 @@ def create_basemap(map_params):
 #    PROCESS
 #=============================================================================
 
+# heading
+print()
+print('============================================================')
+print('                      CREATE BASEMAPS                       ')
+
 maps = [
     ('World [default]',     GEBCO_WORLD,     GEBCO_WORLD_HILLSHADE,     CP_DEFAULT, EPSG_WORLD,     BASEMAP_WORLD            ),
     ('Arctic [default]',    GEBCO_ARCTIC,    GEBCO_ARCTIC_HILLSHADE,    CP_DEFAULT, EPSG_ARCTIC,    BASEMAP_ARCTIC           ),
@@ -100,33 +117,3 @@ cleanup(GEBCO_ANTARCTIC_HILLSHADE)
 """
 Metadata
 """
-
-#=============================================================================
-#    BACKUP
-#=============================================================================
-
-"""
-msg = 'Creating World RGB...'
-cmd = f'gdaldem color-relief {GEBCO_WORLD} {CP_DEFAULT} {GEBCO_WORLD_RGB} -of GTiff -b 1 -compute_edges {GDAL_CREATE_OPTIONS}')
-run(msg,cmd)
-
-msg = 'Creating AWI Basemap World...'
-cmd = f'python3 -m gdal_calc --calc "uint8(((A/255.)*(((B*0.75)+(255.*0.25))/255.))*255)" --format GTiff --type Byte -A {GEBCO_WORLD_RGB} --A_band 1 -B {GEBCO_WORLD_HILLSHADE} --B_band 1 {GDAL_CREATE_OPTIONS_PY} --allBands=A --overwrite --outfile {BASEMAP_WORLD}')
-run(msg,cmd)
-
-msg = 'Creating Arctic RGB...'
-cmd = f'gdaldem color-relief {GEBCO_ARCTIC} {CP_DEFAULT} {GEBCO_ARCTIC_RGB} -of GTiff -b 1 -compute_edges {GDAL_CREATE_OPTIONS}')
-run(msg,cmd)
-
-msg = 'Creating AWI Basemap Arctic...'
-cmd = f'python3 -m gdal_calc --calc "uint8(((A/255.)*(((B*0.75)+(255.*0.25))/255.))*255)" --format GTiff --type Byte -A {GEBCO_ARCTIC_RGB} --A_band 1 -B {GEBCO_ARCTIC_HILLSHADE} --B_band 1 {GDAL_CREATE_OPTIONS_PY} --allBands=A --overwrite --outfile {BASEMAP_ARCTIC}')
-run(msg,cmd)
-
-msg = 'Creating Antarctic RGB...'
-cmd = f'gdaldem color-relief {GEBCO_ANTARCTIC} {CP_DEFAULT} {GEBCO_ANTARCTIC_RGB} -of GTiff -b 1 -compute_edges {GDAL_CREATE_OPTIONS}')
-run(msg,cmd)
-
-msg = 'Creating AWI Basemap Antarctic...'
-cmd = f'python3 -m gdal_calc --calc "uint8(((A/255.)*(((B*0.75)+(255.*0.25))/255.))*255)" --format GTiff --type Byte -A {GEBCO_ANTARCTIC_RGB} --A_band 1 -B {GEBCO_ANTARCTIC_HILLSHADE} --B_band 1 {GDAL_CREATE_OPTIONS_PY} --allBands=A --overwrite --outfile {BASEMAP_ANTARCTIC}')
-run(msg,cmd)
-"""

process/config.py

@@ -9,7 +9,7 @@
 ####################################################################
 
 This script holds the configuration for the AWI Basemap creation
-process. Adjust the settings the below and the other basemap python
+process. Adjust the settings below and the other basemap python
 scripts will source these settings. The following can be adjusted
 here:
 
@@ -47,6 +47,9 @@ import os
 #    INPUT FOLDERS
 #=============================================================================
 
+# These folders are part of the processing infrastructure. Only change the
+# variables if the directory names or locations have been changed.
+
 # path to data directory
 PATH_DATA = os.path.abspath(os.path.join('..','data'))
 
@@ -64,6 +67,10 @@ PATH_RESULT = os.path.abspath(os.path.join('..','result'))
 #    INPUT FILES
 #=============================================================================
 
+# These file paths are using the current version of the input files' names. If
+# any of the input files are updated in the data directory, their filenames
+# need to updated in this section.
+
 # input filepath to ADD Coastlines
 ADD_COASTLINE_IN = os.path.join(PATH_ADD,'add_coastline_medium_res_polygon_v7.3.gpkg')
 # input filepath to ADD Rock Outcrops
@@ -88,28 +95,37 @@ CP_DARK = os.path.abspath(os.path.join('..','styles','color_palettes','basemap_d
 #    GENERAL CREATE OPTIONS
 #=============================================================================
 
+# Here are some general options for the resulting grids like resolution and
+# shading exaggeration. Adjusting the resolution of the output grid will
+# directly affect file sizes. The maximum resolution for the World map is the
+# original GEBCO resolution (15 arc seconds / 0.00416... degree). For the
+# Polar maps, higher resolutions than 500m can be achieved theoretically, but
+# this will result in some ugly artifacts (from warping) in the lower
+# latitudes.
+#
+# The best tradeoff between file size, resolution and artifacts is
+#   World: 30 arc seconds / 0.0083... degree
+#   Polar: 1000 meter
+#
+# The extent of the Polar maps should be kept to an acceptable minium (larger
+# number) to reduce file size and atrifacts from warping in the lower
+# latitudes.
+
 # Final AWI Basemap (World) resolution
 # 15 arc seconds (original GEBCO resolution)
 #RESOLUTION_DEG = 0.004166666666666666
 # 30 arc seconds
 RESOLUTION_DEG = 0.008333333333333333
 
-#RESOLUTION_DEG = 0.025                         # DEBUGGING
-
 # Final AWI Basemap (Arctic and Antarctic) resolution
 # pendant to 15 arc seconds
-RESOLUTION_M = 500.0
+#RESOLUTION_M = 500.0
 # pendant to 30 arc seconds
-#RESOLUTION_M = 1000.0
-
-# number of pixels for interpolation (fill NoData at poles)
-INTERPOLATION = 1 / RESOLUTION_M * 5000         # DEBUGGING
+RESOLUTION_M = 1000.0
 
 # GIMP Resolution
-# 7.5 arc seconds (half the original GEBCO resolution)
-GIMP_RESOLUTION = 0.0020833333333333333
-
-#GIMP_RESOLUTION = 0.01                         # DEBUGGING
+# half the GEBCO resolution
+GIMP_RESOLUTION = RESOLUTION_DEG / 2
 
 # Hillshade z factor / vertical exaggeration
 Z_FACTOR = 10.0
@@ -125,6 +141,16 @@ ANTARCTIC_EXTENT_LAT= -30.0
 #    TEMPORARY FILES
 #=============================================================================
 
+# These are file paths to temporary files that will be created (and cleaned
+# up) during the process. The varibales care used throughout the different
+# scripts.
+# The varibales marked as "DEBUGGING" are for the GEBCO process which needs to
+# be tweaked to avoid the gap at the south and north poles. This is likely due
+# to projecting to polar stereographic with such low latitudes (large
+# extents). Once a nicer solution is found, the variables will be adjusted.
+# This should, however, not influence the general process, but is just an
+# internal change.
+
 # ADD:
 # ADD land with cut rock outcrops
 ADD_ICE_SHEET = os.path.join(PATH_ADD,'add_ice_sheet.gpkg')
@@ -157,27 +183,24 @@ GLIMS_ANTARCTIC = os.path.join(PATH_GLIMS,'glims_antarctic.gpkg')
 # GEBCO World
 GEBCO_WORLD = os.path.join(PATH_GEBCO,'GEBCO_world.tif')
 GEBCO_WORLD_HILLSHADE = os.path.join(PATH_GEBCO,'GEBCO_world_hillshade.tif')
-#GEBCO_WORLD_RGB = os.path.join(PATH_GEBCO,'GEBCO_world_rgb.tif')
 
 # GEBCO Arctic
-GEBCO_ARCTIC_CLIP = os.path.join(PATH_GEBCO,'GEBCO_arctic_clip.tif')
+#GEBCO_ARCTIC_CLIP = os.path.join(PATH_GEBCO,'GEBCO_arctic_clip.tif') # DEBUGGING
 GEBCO_ARCTIC_CLIP_VRT = os.path.join(PATH_GEBCO,'GEBCO_arctic_clip.vrt') # DEBUGGING
-GEBCO_ARCTIC_FULL = os.path.join(PATH_GEBCO,'GEBCO_arctic_full.tif') # DEBUGGING
+#GEBCO_ARCTIC_FULL = os.path.join(PATH_GEBCO,'GEBCO_arctic_full.tif') # DEBUGGING
 GEBCO_ARCTIC_FULL_VRT = os.path.join(PATH_GEBCO,'GEBCO_arctic_full.vrt') # DEBUGGING
 GEBCO_ARCTIC = os.path.join(PATH_GEBCO,'GEBCO_arctic.tif')
 GEBCO_ARCTIC_HILLSHADE = os.path.join(PATH_GEBCO,'GEBCO_arctic_hillshade.tif')
-#GEBCO_ARCTIC_RGB = os.path.join(PATH_GEBCO,'GEBCO_arctic_rgb.tif')
 
 # GEBCO Antarctic
-GEBCO_ANTARCTIC_CLIP = os.path.join(PATH_GEBCO,'GEBCO_antarctic_clip.tif')
+#GEBCO_ANTARCTIC_CLIP = os.path.join(PATH_GEBCO,'GEBCO_antarctic_clip.tif') # DEBUGGING
 GEBCO_ANTARCTIC_CLIP_VRT = os.path.join(PATH_GEBCO,'GEBCO_antarctic_clip.vrt') # DEBUGGING
-GEBCO_ANTARCTIC_FULL = os.path.join(PATH_GEBCO,'GEBCO_antarctic_full.tif') # DEBUGGING
+#GEBCO_ANTARCTIC_FULL = os.path.join(PATH_GEBCO,'GEBCO_antarctic_full.tif') # DEBUGGING
 GEBCO_ANTARCTIC_FULL_VRT = os.path.join(PATH_GEBCO,'GEBCO_antarctic_full.vrt') # DEBUGGING
 GEBCO_ANTARCTIC = os.path.join(PATH_GEBCO,'GEBCO_antarctic.tif')
 GEBCO_ANTARCTIC_HILLSHADE = os.path.join(PATH_GEBCO,'GEBCO_antarctic_hillshade.tif')
-#GEBCO_ANTARCTIC_RGB = os.path.join(PATH_GEBCO,'GEBCO_antarctic_rgb.tif')
 
-# Temporary files
+# temporary files
 TMP_POLAR = os.path.join(PATH_GEBCO,'tmp_polar.tif') # DEBUGGING
 TMP_RGB = os.path.join(PATH_GEBCO,'tmp_rgb.tif')
 TMP_SHADED = os.path.join(PATH_GEBCO,'tmp_shaded.tif')
@@ -205,50 +228,55 @@ BASEMAP_ANTARCTIC_DARK = os.path.join(PATH_RESULT,'AWI_Basemap_Antarctic_dark.ti
 #    PROJECTIONS
 #=============================================================================
 
+# The EPSG codes for the three map projections only need to be changed, if a
+# different projection is required for the Map Viewers.
+
 # EPSG code for World Basemap
-EPSG_WORLD = 'EPSG:4326'
+EPSG_WORLD = 'EPSG:4326' # WGS84 / Geographic
 
 # EPSG code for Arctic Basemap
-EPSG_ARCTIC = 'EPSG:3995'
+EPSG_ARCTIC = 'EPSG:3995' # WGS 84 / Arctic Polar Stereographic
 
 # EPSG code for Antarctic Basemap
-EPSG_ANTARCTIC = 'EPSG:3031'
+EPSG_ANTARCTIC = 'EPSG:3031' # WGS 84 / Antarctic Polar Stereographic
 
 
 #=============================================================================
 #    OPERATIONAL PARAMETERS
 #=============================================================================
 
+# The following settings are operational parameters for the process. If, e.g.
+# your Python (v3.x) installation has a different executable, like "python3"
+# as it comes with the OSGeo4W Python installation, some scripts need to know
+# that and the command needs to be changed here.
+# And if the temporary GeoTIFFs created by GDAL shall be created with
+# different create options, they can be adjusted below. The create options for
+# the final results, however, are specified in the last python script.
+
 # Python executable
 PYTHON = 'python3'
 
 # GDAL creation options
+GDAL_CREATE_OPTIONS = '-co COMPRESS=NONE -co BIGTIFF=IF_NEEDED'
 #GDAL_CREATE_OPTIONS = '-co COMPRESS=DEFLATE -co PREDICTOR=2 -co ZLEVEL=9 -co BIGTIFF=YES'
-GDAL_CREATE_OPTIONS = ''
 
 # GDAL creation options for python modules
+GDAL_CREATE_OPTIONS_PY = '--co COMPRESS=NONE --co BIGTIFF=IF_NEEDED'
 #GDAL_CREATE_OPTIONS_PY = '--co COMPRESS=DEFLATE --co PREDICTOR=2 --co ZLEVEL=9 --co BIGTIFF=YES'
-GDAL_CREATE_OPTIONS_PY = ''
-
-# Warp options for gdalwarp
-SOURCE_EXTRA = 1500
-MEM = 750
-GDAL_CACHEMAX = 9000
-WARP_OPTIONS = f'-wo SOURCE_EXTRA={SOURCE_EXTRA} -wm {MEM} -multi -wo NUM_THTEADS=ALL_CGUS -et 0'
-
-# Method to create overviews
-OVERVIEW_METHOD = 'cubic'
 
 
 #=============================================================================
 #    RUN FUNCTION
 #=============================================================================
 
+# function to run a shell command and print a message to stdout
 def run(msg,cmd):
-    print('===================================================')
+    print('============================================================')
     print()
     print(msg)
     print()
+    print(cmd)
+    print()
     os.system(cmd)
     print()
 
@@ -257,15 +285,17 @@ def run(msg,cmd):
 #    CLEANUP
 #=============================================================================
 
+# list of temporary files that shall not be deleted
 DONT_DELETE = [
-    GEBCO_WORLD,
-    GEBCO_WORLD_HILLSHADE,
-    GEBCO_ARCTIC,
-    GEBCO_ARCTIC_HILLSHADE,
-    GEBCO_ANTARCTIC,
-    GEBCO_ANTARCTIC_HILLSHADE
+#    GEBCO_WORLD,
+#    GEBCO_WORLD_HILLSHADE,
+#    GEBCO_ARCTIC,
+#    GEBCO_ARCTIC_HILLSHADE,
+#    GEBCO_ANTARCTIC,
+#    GEBCO_ANTARCTIC_HILLSHADE
 ]
 
+# function to cleanup temporary files if they are not in the DONT_DELETE list
 def cleanup(dataset):
     if dataset not in DONT_DELETE:
         if os.path.isfile(dataset):

process/run.bat

 @echo off
 
-python3 1_process_add.py
-python3 2_process_gimp.py
-python3 3_process_glims.py
-python3 4_process_gebco.py
-python3 5_create_shading_layers.py
-python3 6_burn_vector_layers.py
-python3 7_create_basemaps.py
+rem PYTHON executable
+set PYTHON=python3
+
+%PYTHON% 1_process_add.py
+%PYTHON% 2_process_gimp.py
+%PYTHON% 3_process_glims.py
+%PYTHON% 4_process_gebco.py
+%PYTHON% 5_create_shading_layers.py
+%PYTHON% 6_burn_vector_layers.py
+%PYTHON% 7_create_basemaps.py
+
+echo.
+echo All done!
+echo.
+set /p DONE=Press Enter to exit!
+echo.
\ No newline at end of file

styles/STYLES.md

+# Styles
+
+As mentioned in the README/Description, the AWI Basemap colors are based on the AWI blues (RGB bright: 7/172/231, RGB dark: 0/62/110). While the default version tones down the blues in their saturation (half), the bright version uses them directly. The dark version uses the blues in a quarter saturation to get an unobtrusive map, great to be used as a background to bring all kinds of primary datasets in the foreground. The land on the other side is held in a grey gradient for all versions. Ice covered areas are shown in white while Antarctic shelf ice is visualized with a light grey.
+
+## This is how the color bars look like:
+
+![AWI Basemap Color Bar Default](https://gitlab.awi.de/sdreutte/basemap/-/raw/master/styles/basemap_default_colorbar.png)
+
+![AWI Basemap Color Bar Bright](https://gitlab.awi.de/sdreutte/basemap/-/raw/master/styles/basemap_bright_colorbar.png)
+
+![AWI Basemap Color Bar Dark](https://gitlab.awi.de/sdreutte/basemap/-/raw/master/styles/basemap_dark_colorbar.png)
+
+## RGB
+
+| Elevation | Default RGB | Bright RGB  | Dark RGB    |
+|:---------:|:-----------:|:-----------:|:-----------:|
+| -12000    | 32/50/64    | 0/36/64     | 48/57/64    |
+| -6000     | 55/86/110   | 0/62/110    | 82/98/110   |
+| -1000     | 101/179/204 | 7/172/231   | 153/191/204 |
+| -10       | 206/233/242 | 191/239/255 | 225/234/237 |
+| -4        | 255/255/255 | 255/255/255 | 255/255/255 |
+| 5         | 230/230/230 | 230/230/230 | 230/230/230 |
+| 100       | 204/204/204 | 204/204/204 | 204/204/204 |
+| 1000      | 179/179/179 | 179/179/179 | 179/179/179 |
+| 10000     | 153/153/153 | 153/153/153 | 153/153/153 |
+| 20000     | 242/242/242 | 242/242/242 | 242/242/242 |
+| 30000     | 255/255/255 | 255/255/255 | 255/255/255 |