Skip to content

GitLab

Commit 2aa5b4f6 authored by Andreas Walter's avatar Andreas Walter 🤙
Browse files

SDI-1189 Merge branch 'release/0.0.3'

parents f4dd44e5 548e8e6e
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 2aa5b4f6
......@@ -9,16 +9,9 @@ ServCat/ServCat.egg-info/*
# Test Scripts
#ServCat/MetadataHarvester/Unittests
# Test Data
recipes/update_pangaea_bathymetry_dimensions.py
# Output Files
*.tab
# PyCache
__pycache__/
*/__pycache__/
......
ServCat/ServCat/MetadataHarvester/DescribeRequest.py
View file @ 2aa5b4f6
......@@ -40,6 +40,17 @@
__all_ = ['DescribeRequest', 'DescribeRequestESRI']
# -------------------------------------
# Import of Python Modules
# -------------------------------------
from typing import Union
# -------------------------------------
# Import of o2_spatial Modules
# -------------------------------------
# from ServCat.Utils import simple_logger as logger
class DescribeRequest(dict):
ows_describe_types = {
......@@ -95,6 +106,7 @@ class DescribeRequest(dict):
service_version: str = '1.1.1',
output_format: str = 'application/json',
**kwargs):
super(DescribeRequest, self).__init__()
service_type = service_type.upper()
self.ows_describe_type = self.ows_describe_types.get(service_type, {}).get(service_version, {})
......@@ -156,6 +168,7 @@ class DescribeRequestESRI(dict):
@param service_url: URL of the ESRI ArcGIS for Server endpoint.
@param server_type: One of the following is possible: ['MapServer', 'ImageServer', 'FeatureServer']
"""
super(DescribeRequestESRI, self).__init__()
self.url = service_url
self.service_type, self.server_type = self.identify()
......@@ -202,26 +215,20 @@ class DescribeRequestESRI(dict):
if 'featureserver' in lower_url:
raise AttributeError("Don't have clue what to do with a 'FeatureServer' ... YET!")
def translate_to(self, service_type: str = 'WMS') -> str:
def translate_to(self, service_type: str = 'WMS') -> None:
service_url = self.url
# if self.service_type == service_type:
# print(f'AA: {self.service_url}')
# return
if 'services/projects' in self.url.lower() and service_type.upper() in ('WMS', 'WFS', 'WCS', 'IMS'):
# ows_url = self.service_url.lower().replace(self.service_type.lower(), '').strip('/')
ows_url = self.url.lower().replace('/wfs', '').replace('/wcs', '').replace('/ims', '').replace(
'/wms', '')
# print(f'A: {ows_url}')
# print(f'A: {"/".join([ows_url, service_type.lower()])}')
# print(self.service_url)
# print(ows_url)
self.url = '/'.join([ows_url, service_type.lower()])
self.service_type = service_type
elif 'services/projects' in self.url.lower() and service_type.upper() == 'REST':
# print(f'B: {self.service_url}')
# TODO Right now our ESRI REST API is only working for URLs with '/wms' at the end,
# e.g. 'HOST/services/projects/reklim/arctic_coastal_dynamics/wms'
# For URLs with different endings, e.g. 'HOST/services/projects/reklim/arctic_coastal_dynamics/wfs',
......@@ -237,7 +244,6 @@ class DescribeRequestESRI(dict):
self.service_type = service_type
elif service_type.upper() in ('WMS', 'WFS', 'WCS'):
# print(f'C: {self.service_url}')
# ows_url = service_url.replace('server/rest/services', 'server/services')
ows_url = service_url.replace('/rest/services/', '/services/')
clean_ows_url = ows_url[:ows_url.index(self.server_type)].strip('/')
......@@ -259,10 +265,9 @@ class DescribeRequestESRI(dict):
self.url = '/'.join(rest_url.split('/')[:-1])
self.service_type = service_type
else:
# print(f'Z: {self.service_url}')
return
def to_request(self, esri_id: str = '') -> str:
def to_request(self, esri_id: Union[str, None] = '') -> str:
"""
@param esri_id: '*' = All Layer Information;
......@@ -310,7 +315,7 @@ if __name__ == '__main__':
# LIMIT=50
)
print(describe.to_request())
# logger.info(describe.to_request())
# describe_esri = DescribeRequestESRI(
# # service_url='https://maps.awi.de/services/projects/reklim/arctic_coastal_dynamics/wfs'
# # service_url='https://maps.awi.de/services/projects/reklim/arctic_coastal_dynamics/wms'
......@@ -318,6 +323,6 @@ if __name__ == '__main__':
# # service_url='https://maps.awi.de/services/projects/globpermafrost/active_layer_thickness_max/ims'
# )
# describe_esri.translate_to('REST')
# print(describe_esri.to_request(esri_id='*'))
# logger.info(describe_esri.to_request(esri_id='*'))
# describe_esri.translate_to('WFS')
# print(describe_esri.to_request())
# logger.info(describe_esri.to_request())
ServCat/ServCat/MetadataHarvester/DescribeRequestESRI.py
View file @ 2aa5b4f6
......@@ -37,146 +37,21 @@
| Created on 26.01.2021
| @author: awalter
"""
# -------------------------------------
# Import of Python Modules
# -------------------------------------
import requests
from ServCat.Utils import DotDict
from ServCat.MetadataHarvester.DescribeRequest import DescribeRequestESRI
logger = DotDict({
'info': lambda x: print(f'| Info: {x}'),
'warning': lambda x: print(f'| Warning: {x}'),
'error': lambda x: print(f'| Error: {x}'),
})
from requests import Response
from typing import Union, Dict, Tuple
# class ESRIUrl:
# __server_types = {
# 'MapServer': {'WMS': 'WMSServer',
# 'WFS': 'WFSServer',
# 'WCS': 'WCSServer',
# 'REST': ''},
# 'ImageServer': {'WMS': 'WMSServer',
# 'WCS': 'WCSServer',
# 'REST': ''},
# 'FeatureServer': {'WFS': 'WFSServer',
# 'REST': ''
# }
# }
#
# def __init__(self, service_url: str):
# """
#
# @param service_url: URL of the ESRI ArcGIS for Server endpoint.
# @param server_type: One of the following is possible: ['MapServer', 'ImageServer', 'FeatureServer']
# """
# self.service_url = service_url
# self.service_type, self.server_type = self.identify()
#
# def identify(self):
# lower_url = self.service_url.lower()
#
# if 'rest' in lower_url:
# return 'REST', self.service_url.split('/')[-1]
#
# if 'services/projects' in lower_url:
# if 'wms' in lower_url:
# return 'WMS', 'MapServer'
# elif 'wfs' in lower_url:
# return 'WFS', 'MapServer'
# elif 'ims' in lower_url:
# return 'WMS', 'ImageServer'
#
# if 'mapserver' in lower_url:
# if 'wmsserver' in lower_url:
# return 'WMS', 'MapServer'
# if 'wfsserver' in lower_url:
# return 'WFS', 'MapServer'
# if 'wcsserver' in lower_url:
# return 'WCS', 'MapServer'
#
# if 'imageserver' in lower_url:
# if 'wmsserver' in lower_url:
# return 'WMS', 'ImageServer'
# if 'wcsserver' in lower_url:
# return 'WCS', 'ImageServer'
#
# if 'featureserver' in lower_url:
# raise AttributeError("Don't have clue what to do with a 'FeatureServer' ... YET!")
#
# def translate_to(self, service_type: str = 'WMS') -> str:
# service_url = self.service_url
#
# # if self.service_type == service_type:
# # print(f'AA: {self.service_url}')
# # return
#
# if 'services/projects' in self.service_url.lower() and service_type.upper() in ('WMS', 'WFS', 'WCS', 'IMS'):
# # ows_url = self.service_url.lower().replace(self.service_type.lower(), '').strip('/')
# ows_url = self.service_url.lower().replace('/wfs', '').replace('/wcs', '').replace('/ims', '').replace(
# '/wms', '')
# # print(f'A: {ows_url}')
# # print(f'A: {"/".join([ows_url, service_type.lower()])}')
# # print(self.service_url)
# # print(ows_url)
# self.service_url = '/'.join([ows_url, service_type.lower()])
# self.service_type = service_type
#
# elif 'services/projects' in self.service_url.lower() and service_type.upper() == 'REST':
# # print(f'B: {self.service_url}')
# self.service_type = service_type
#
# elif service_type.upper() in ('WMS', 'WFS', 'WCS'):
# # print(f'C: {self.service_url}')
# ows_url = service_url.replace('server/rest/services', 'server/services')
# clean_ows_url = ows_url[:ows_url.index(self.server_type)].strip('/')
#
# service_types = self.__server_types.get(self.server_type, 'MapServer')
# if service_type not in service_types:
# raise AttributeError(f'"{service_type}" is not supported!')
#
# ows_type = service_types.get(service_type, 'WMS')
#
# self.service_url = '/'.join([clean_ows_url, self.server_type, ows_type])
# self.service_type = service_type
#
# elif service_type == 'REST':
# if self.service_type == service_type:
# return
# rest_url = service_url.replace('server/services', 'server/rest/services')
# self.service_url = '/'.join(rest_url.split('/')[:-1])
# self.service_type = service_type
# else:
# # print(f'Z: {self.service_url}')
# return
#
# def to_request(self, esri_id: str = '') -> str:
# """
#
# @param esri_id: '*' = All Layer Information;
# [0-999] = Specific Layer Information;
# '' and None = Service Information;
# @return:
# """
# if esri_id is None:
# esri_id = ''
#
# if self.service_type == 'REST' and esri_id == '*':
# return f'{self.service_url}/layers?f=json'
#
# if self.service_type == 'REST' and esri_id == '':
# return f'{self.service_url}?f=json'
#
# elif self.service_type == 'REST':
#
# if str(esri_id):
# esri_id = f"/{str(esri_id).strip('/')}"
#
# return f'{self.service_url}{esri_id}?f=json'
# else:
# return f'{self.service_url}'
# -------------------------------------
# Import of o2_spatial Modules
# -------------------------------------
from ServCat.Utils import simple_logger as logger
from ServCat.MetadataHarvester.DescribeRequest import DescribeRequestESRI
def __get(request_url: str, content_type: str) -> dict:
def __get(request_url: str, content_type: str) -> Union[Dict, Response]:
content_types = {
'json': {"Content-Type": "application/json"},
'xml': {"Content-Type": "text/xml"},
......@@ -198,7 +73,7 @@ def __get(request_url: str, content_type: str) -> dict:
return response
def describe_content(url: str = None) -> (str, dict):
def describe_content(url: str = None) -> Tuple[str, Dict]:
esri = DescribeRequestESRI(service_url=url)
esri.translate_to(service_type='REST')
request_service = esri.to_request(esri_id=None)
......@@ -250,8 +125,8 @@ if __name__ == '__main__':
for url in URLs:
# describe = ESRIUrl(service_url=url)
print('-' * 40)
# print(f'REQUEST: {describe.to_request()}')
logger.info('-' * 40)
# logger.info(f'REQUEST: {describe.to_request()}')
request, content = describe_content(url)
# for ows in ('REST', 'WMS', 'REST', 'WCS', 'REST', 'WFS', 'REST'):
......@@ -263,10 +138,10 @@ if __name__ == '__main__':
# for ows in ('REST',):
# try:
# # describe.translate_to(ows)
# # print(f'Translated to {ows}: {describe.to_request()}')
# # print(f'Translated to {ows}: {describe.to_request(esri_id="")}')
# # print(f'Translated to {ows}: {describe.to_request(esri_id=None)}')
# # print(f'Translated to {ows}: {describe.to_request(esri_id="*")}')
# # print(f'Translated to {ows}: {describe.to_request(esri_id=0)}') # AWI internal: Not supported from our proxy server, yet
# # logger.info(f'Translated to {ows}: {describe.to_request()}')
# # logger.info(f'Translated to {ows}: {describe.to_request(esri_id="")}')
# # logger.info(f'Translated to {ows}: {describe.to_request(esri_id=None)}')
# # logger.info(f'Translated to {ows}: {describe.to_request(esri_id="*")}')
# # logger.info(f'Translated to {ows}: {describe.to_request(esri_id=0)}') # AWI internal: Not supported from our proxy server, yet
# except Exception as err:
# print(err)
# logger.error(err)
ServCat/ServCat/MetadataHarvester/ESRI/ImageServer.py
View file @ 2aa5b4f6
......@@ -53,18 +53,11 @@ from typing import Dict, AnyStr, List, Callable, Tuple
# -------------------------------------
# Import of o2_spatial Modules
# -------------------------------------
from ServCat.Utils import DotDict
from ServCat.Utils import simple_logger as logger
from ServCat.Utils import Layer, Service, LayerAttributeField, Contact
from ServCat.MetadataHarvester.AbstractHarvester import AbstractHarvester
from ServCat.MetadataHarvester.Utils import get_default_timestamp, transform_bounding_box
# -------------------------------------
logger = DotDict({
'info': lambda x: print(f'| Info: {x}'),
'warning': lambda x: print(f'| Warning: {x}'),
'error': lambda x: print(f'| Error: {x}'),
})
class WMSServer(AbstractHarvester):
......
ServCat/ServCat/MetadataHarvester/ESRI/MapServer.py
View file @ 2aa5b4f6
......@@ -60,14 +60,8 @@ from ServCat.MetadataHarvester.AbstractHarvester import AbstractHarvester
from ServCat.MetadataHarvester.WMS import WebMapService
from ServCat.MetadataHarvester.Utils import get_default_timestamp, save_get, transform_bounding_box, proofed_get
from ServCat.Utils import Service, Layer, LayerAttributeField, Contact
from ServCat.Utils import DotDict
from ServCat.Utils import simple_logger as logger
# -------------------------------------
logger = DotDict({
'info': lambda x: print(f'| Info: {x}'),
'warning': lambda x: print(f'| Warning: {x}'),
'error': lambda x: print(f'| Error: {x}'),
})
chars = re.compile(f'[^-_+.{digits}{ascii_letters}]')
......@@ -191,16 +185,21 @@ class WMSServer(AbstractHarvester):
# Helper Function
def scanner(obj: Dict, parent_name: str) -> None:
if obj.get("Title") != 'Layers':
layer_structure.setdefault(parent_name, []).append({'name': obj.get("Name", ""), 'title': obj.get("Title", "")})
layer_structure.setdefault(parent_name, []).append({
# Use 'Title' tag as name if 'Name' tag not available
# 'ESRI Layer Groups' do not have a 'Name' tag (since ArcGIS Server 10.8.X)!
'name': obj.get("Name", obj.get("Title", "")),
'title': obj.get("Title", ""),
})
layer_obj = obj.get('Layer', None)
if layer_obj:
parent_name_new = obj.get("Title")
if isinstance(layer_obj, dict):
scanner(layer_obj, parent_name_new)
scanner(obj=layer_obj, parent_name=parent_name_new)
elif isinstance(layer_obj, list):
for item in layer_obj:
scanner(item, parent_name_new)
scanner(obj=item, parent_name=parent_name_new)
layer_structure = {}
scanner(obj=capabilities.get('Capability', {}).get('Layer', {}), parent_name='#')
......@@ -231,6 +230,8 @@ class WMSServer(AbstractHarvester):
service_content_entry = self.service.contents.get(layer_name, {})
elif isinstance(self.service.contents, WebMapService):
service_content_entry = proofed_get(self.service.contents, layer_name, {})
else:
raise AttributeError('GetCapabilities document is not available!')
bounded_by = proofed_get(service_content_entry, 'boundingBoxWGS84', [-180.0, -90, 180, 90])
......@@ -511,7 +512,6 @@ class WFSServer(AbstractHarvester):
for x in feature_type:
layers.update({x.get('wfs:Name'): x.get('wfs:Title')})
elif isinstance(feature_type, OrderedDict):
print(feature_type)
layers.update({feature_type.get('wfs:Name'): feature_type.get('wfs:Title')})
layer_titles = {
......@@ -533,10 +533,7 @@ class WFSServer(AbstractHarvester):
if layer.get('type', '') == 'Group Layer':
return
# print(f"Name: {layer.get('name', '')}")
# title = layer.get('title', '')
# print(f"Title: {title} {title}")
# copyright = layer.get('copyrightText', '')
......@@ -662,7 +659,6 @@ class WFSServer(AbstractHarvester):
if not esri_layer:
continue
# a, b = esri_layer
# print('--', b.get('name'))
# b['title'] = self.layer_titles.get(b.get('name'))
esri_layers.setdefault(*esri_layer)
......@@ -678,7 +674,6 @@ class WFSServer(AbstractHarvester):
layers = self.merge_layers()
# print(f'Capa: {self.capabilities}')
bounded_by, crs_options = _get_merged_spatial_information(self.describe_dict.get('fullExtent', {}))
servcat_service = Service(
......@@ -799,7 +794,7 @@ if __name__ == '__main__':
#
servcat_json_wms = wms.to_servcat_json()
# capa = wms.parse_layer_structure()
# print(capa)
# wfs = WFSServer(url='https://maps.awi.de/services/projects/fram/ofos/wfs',
# wfs = WFSServer(url='https://coastmap.hzg.de/server/services/NOAH_geoDB/BSH_CB153/MapServer/WFSServer',
......
ServCat/ServCat/MetadataHarvester/ESRI/__init__.py
View file @ 2aa5b4f6
......@@ -49,11 +49,12 @@ __credits__ = 'Alfred Wegener Institute > Computer and Data Center > Software En
import json
import requests
from pathlib import Path
from typing import List, Dict
from typing import List, Dict, Union
# --------------------------
# Import of self-written Modules
# --------------------------
from ServCat.Utils import simple_logger as logger
from ServCat.MetadataHarvester.DescribeRequest import DescribeRequestESRI
......@@ -130,22 +131,22 @@ class EsriRESTHarvester:
"""
url, query = self.url.split('?')
if not folders:
print(f'{url} :: No folders!')
logger.error(f'{url} :: No folders!')
return
for folder in iter(folders):
if self.known_folder_names and folder not in self.known_folder_names:
print(f'Folder: {folder}/ (excluded)')
logger.warning(f'Folder: {folder}/ (excluded)')
continue
print(f'Folder: {folder}/')
logger.info(f'Folder: {folder}/')
_folders, _services = self.__get_folder_and_service_names(f'{url}/{folder}?f=json')
self.__for_each_service(_services)
print('-'*80)
logger.info('-'*80)
def __for_each_service(self, services: List[dict]) -> None:
"""
......@@ -167,7 +168,7 @@ class EsriRESTHarvester:
url, query = self.url.split('?')
if not services:
print(f'{url} :: No services!')
logger.error(f'{url} :: No services!')
return
for service in iter(services):
......@@ -175,7 +176,7 @@ class EsriRESTHarvester:
esri_service_type = service.get('type', None)
# if len(self.OWServices) >= 10:
# # print(len(self.OWServices))
# # logger.info(len(self.OWServices))
# return
if not name or not esri_service_type:
......@@ -187,22 +188,21 @@ class EsriRESTHarvester:
continue
entry_url = f'{url}/{name}/{esri_service_type}?f=json'
# print(f'Request URL: {entry_url}')
# logger.info(f'Request URL: {entry_url}')
entry_json = self.__get(entry_url)
layers = entry_json.get('layers', [])
if not layers and esri_service_type.lower() == 'mapserver':
print('No Layers found! SKIP')
logger.warning('No Layers found! SKIP')
continue
esri = DescribeRequestESRI(service_url=f'{url}/{name}/{esri_service_type}')
# print(name)
if esri_service_type.lower() == 'imageserver':
# esri = DescribeRequestESRI(service_url=f'https://maps.awi.de/services/projects/{name}/ims')
esri.translate_to(service_type='WMS')
print(f'Service: {name} :: {esri_service_type} :: WMSServer')
print(esri_service_type, entry_json.get('supportedExtensions', '').split(', '))
logger.info(f'Service: {name} :: {esri_service_type} :: WMSServer')
logger.info(esri_service_type, entry_json.get('supportedExtensions', '').split(', '))
self.OWServices.append({
'software': 'ESRI',
......@@ -211,20 +211,18 @@ class EsriRESTHarvester:
})
elif esri_service_type.lower() == 'mapserver':
# print(esri_service_type, entry_json.get('supportedExtensions', '').split(','))
# esri = DescribeRequestESRI(service_url=f'https://maps.awi.de/services/projects/{name}/wms')
# print(entry_json.get('supportedExtensions', ''))
supported_extensions = entry_json.get('supportedExtensions', '').split(',')
for server_type in supported_extensions:
server_type_ = server_type.strip().lower()
if server_type_ not in ('wmsserver', 'wfsserver', ):
continue
print(f'Service: {name} :: {esri_service_type} :: {server_type}')
logger.info(f'Service: {name} :: {esri_service_type} :: {server_type}')
if server_type_ == 'wmsserver':
esri.translate_to(service_type='WMS')
# print(f' -->> {esri.to_request()}')
self.OWServices.append({
'software': 'ESRI',
'types': {'WMS': '1.1.1'},
......@@ -232,7 +230,6 @@ class EsriRESTHarvester:
})
elif server_type_ == 'wfsserver':
esri.translate_to(service_type='WFS')
# print(f' -->> {esri.to_request()}')
self.OWServices.append({
'software': 'ESRI',
'types': {'WFS': '2.0.0'},
......
ServCat/ServCat/MetadataHarvester/Utils.py
View file @ 2aa5b4f6
......@@ -37,11 +37,20 @@
| Created on 03.12.2020
| @author: awalter
"""
# -------------------------------------
# Import of Python Modules
# -------------------------------------
import datetime
import requests
from osgeo import ogr, osr, gdal
from typing import AnyStr, List, Tuple, Dict, NewType, Union
# -------------------------------------
# Import of o2_spatial Modules
# -------------------------------------
from ServCat.Utils import simple_logger as logger
def proofed_get(to_proof, name: str, default: Union[str, Dict, List] = '') -> Union[str, Dict, List]:
"""
......@@ -221,7 +230,7 @@ def transform_bounding_box(minx: str, maxx: str, miny: str, maxy: str, source: i
point_max.Transform(transform)
maxy, maxx, maxz = point_max.GetPoint()
except Exception as err:
print(f'| GDAL Transformation Error: {err}')
logger.error(f'| GDAL Transformation Error: {err}')
# re-ordering bbox after re-projecting to make sure minx/miny is not greater than maxx/maxy
miny, maxy = sorted((miny, maxy))
......@@ -244,9 +253,10 @@ def gdal_error_handler(err_class, err_num, err_msg):
}
err_msg = err_msg.replace('\n', ' ')
err_class = errtype.get(err_class, 'None')
print(f'Error Number: {err_num}')
print(f'Error Type: {err_class}')
print(f'Error Message: {err_msg}')
logger.error(f'GDAL ERROR:'
f'\nError Number: {err_num}'
f'\nError Type: {err_class}'
f'\nError Message: {err_msg}')
if __name__ == '__main__':
......@@ -258,7 +268,7 @@ if __name__ == '__main__':
# point0 = {'minX': '-180.0', 'minY': 0.0, 'maxX': 180.0, 'maxY': 89.99}
point0 = {'minX': '-1.0', 'minY': 0.0, 'maxX': 180.0, 'maxY': 89.99}
print(point0)
logger.info(point0)
point2 = transform_bounding_box(maxx=point0.get('maxX'),
maxy=point0.get('maxY'),
......@@ -266,7 +276,7 @@ if __name__ == '__main__':
miny=point0.get('minY'),
source=4326,
target=3995)
print(point2)
logger.info(point2)
point3 = transform_bounding_box(maxx=point2.get('maxX'),
maxy=point2.get('maxY'),
......@@ -275,7 +285,7 @@ if __name__ == '__main__':
source=3995,
target=4326)
print(point3)
logger.info(point3)