Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Thresholds #384

Draft
wants to merge 16 commits into
base: main
Choose a base branch
from
Draft
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def __init__(
self,
layer: Layer,
feature: Feature,
thresholds: Optional[Tuple[float, float, float, float]],
thresholds: Optional[Tuple[float, float, float, float]] = None,
) -> None:
if thresholds is None:
thresholds = (0.2, 0.5, 0.8, 0.9)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ def __init__(
self,
layer: Layer,
feature: geojson.Feature,
thresholds: Optional[Tuple[float, float, float, float]],
thresholds: Optional[Tuple[float, float, float, float]] = None,
) -> None:
super().__init__(layer=layer, feature=feature)
self.threshold_4 = 2
self.threshold_3 = 3
self.threshold_2 = 4
self.threshold_1 = 8
self.threshold_4 = self.thresholds[0]
self.threshold_3 = self.thresholds[1]
self.threshold_2 = self.thresholds[2]
self.threshold_1 = self.thresholds[3]
self.element_count = None
self.contributions_sum = None
self.contributions_rel = {} # yearly interval
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,26 @@ Currentness:
result_description: |
Over 50% of the $elements features ($layer_name) were edited $years.
layer-thresholds:
default: [ 0.2, null, 0.6, null]
default: [ 2, 3, 4, 8]
amenities: null
building_count: null
major_roads_count: null
infrastructure_lines: null
poi: null
jrc_airport_count: null
jrc_bridge_count: null
jrc_cultural_heritage_site_count: null
jrc_education_count: null
jrc_health_count: null
jrc_mass_gathering_sites_count: null
jrc_power_generation_plant_count: null
jrc_railway_count: null
jrc_road_count: null
jrc_water_treatment_plant_count: null
Comment on lines +28 to +37
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think those layers are not beein used at the moment. Lets not add them here for now.

lulc: null
major_roads_count: null
mapaction_lakes_count: null
mapaction_major_roads_length: null
mapaction_rail_length: null
mapaction_rivers_length: null
mapaction_settlements_count: null
Comment on lines +40 to +44
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above comment

poi: null

Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
from io import StringIO
from string import Template
from typing import Optional, Tuple

import dateutil.parser
import matplotlib.pyplot as plt
Expand All @@ -19,7 +20,10 @@ class GhsPopComparisonBuildings(BaseIndicator):
"""Set number of features and population into perspective."""

def __init__(
self, layer: Layer, feature: Feature, thresholds: tuple = None
self,
layer: Layer,
feature: Feature,
thresholds: Optional[Tuple[float, float, float, float]] = None,
) -> None:
super().__init__(layer=layer, feature=feature, thresholds=thresholds)
# Those attributes will be set during lifecycle of the object.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
from io import StringIO
from string import Template
from typing import Optional, Tuple

import dateutil.parser
import matplotlib.pyplot as plt
Expand All @@ -19,7 +20,10 @@ class GhsPopComparisonRoads(BaseIndicator):
"""Set number of features and population into perspective."""

def __init__(
self, layer: Layer, feature: Feature, thresholds: tuple = None
self,
layer: Layer,
feature: Feature,
thresholds: Optional[Tuple[float, float, float, float]] = None,
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For this indicator its actualy a tuple of dicts (see in the layer-thresholds in the metadata:
[big-data/ohsome](default: [{ a: 1000 }, null, { a: 500 }, null])

Here we also need to check the threshold function used by this indicator to use the threshold function parameter

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

) -> None:
super().__init__(layer=layer, feature=feature, thresholds=thresholds)
# Those attributes will be set during lifecycle of the object.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ GhsPopComparisonRoads:
$feature_length_per_sqkm km of roads per sqkm mapped.
layer-thresholds:
default: [{ a: 1000 }, null, { a: 500 }, null]
major_roads_length: null
jrc_road_length: null
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets not include JRC layers here

major_roads_length: null
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def __init__(
self,
layer: Layer,
feature: Feature,
thresholds: Optional[Tuple[float, float, float, float]],
thresholds: Optional[Tuple[float, float, float, float]] = None,
time_range: str = "2008-01-01//P1M",
) -> None:
super().__init__(layer=layer, feature=feature)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,16 @@ MappingSaturation:
amenities: null
building_count: null
infrastructure_lines: null
jrc_education_count: null
jrc_health_count: null
jrc_mass_gathering_sites_count: null
jrc_railway_length: null
jrc_road_length: null
Comment on lines +23 to +27
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as before

lulc: null
major_roads_length: null
poi, thresholds: null
mapaction_lakes_area: null
mapaction_major_roads_length: null
mapaction_rail_length: null
mapaction_rivers_length: null
mapaction_settlements_count: null
Comment on lines +30 to +34
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as before

poi: null
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""An Indicator for testing purposes."""
from string import Template
from typing import Optional, Tuple

import dateutil.parser
from geojson import Feature
Expand All @@ -11,7 +12,10 @@

class Minimal(BaseIndicator):
def __init__(
self, layer: Layer, feature: Feature, thresholds: tuple = None
self,
layer: Layer,
feature: Feature,
thresholds: Optional[Tuple[float, float, float, float]] = None,
) -> None:
super().__init__(layer=layer, feature=feature, thresholds=thresholds)
self.count = 0
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
from io import StringIO
from string import Template
from typing import Optional, Tuple

import dateutil.parser
import matplotlib.pyplot as plt
Expand All @@ -18,7 +19,10 @@

class PoiDensity(BaseIndicator):
def __init__(
self, layer: Layer, feature: Feature, thresholds: tuple = None
self,
layer: Layer,
feature: Feature,
thresholds: Optional[Tuple[float, float, float, float]] = None,
) -> None:
super().__init__(layer=layer, feature=feature, thresholds=thresholds)
self.threshold_yellow = 30
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
from io import StringIO
from string import Template
from typing import Optional, Tuple

import dateutil.parser
import matplotlib.patches as mpatches
Expand All @@ -14,7 +15,10 @@

class TagsRatio(BaseIndicator):
def __init__(
self, layer: Layer, feature: Feature, thresholds: tuple = None
self,
layer: Layer,
feature: Feature,
thresholds: Optional[Tuple[float, float, float, float]] = None,
) -> None:
super().__init__(layer=layer, feature=feature, thresholds=thresholds)
self.threshold_yellow = 0.75
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,12 @@ TagsRatio:
layer-thresholds:
default: [ 0.25, null, 0.75, null]
building_count: null
jrc_airport_count: null
jrc_bridge_count: null
jrc_cultural_heritage_site_count: null
jrc_education_count: null
jrc_health_count: null
jrc_mass_gathering_sites_count: null
jrc_power_generation_plant_count: null
jrc_road_length: null
Comment on lines +23 to +30
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as before

major_roads_length: null
2 changes: 1 addition & 1 deletion workers/ohsome_quality_analyst/oqt.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ async def _(
logging.info("Layer name: {0:4}".format(layer.name))

indicator_class = name_to_class(class_type="indicator", name=name)
indicator = indicator_class(layer, feature, thresholds=None)
indicator = indicator_class(layer, feature)

logging.info("Run preprocessing")
await indicator.preprocess()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ def test_invalid_set_of_arguments(self):
"bpolys": bpolys,
"dataset": "foo",
"featureId": "3",
"thresholds": None,
}
response = self.client.post(self.endpoint, json=parameters)
self.assertEqual(response.status_code, 422)
Expand Down Expand Up @@ -196,7 +195,6 @@ def test_indicator_layer_data(self):
]
},
},
"thresholds": None,
}
response = self.client.post(self.endpoint, json=parameters)
self.run_tests(response, (self.general_schema, self.feature_schema))
Expand All @@ -210,7 +208,6 @@ def test_indicator_layer_data_invalid(self):
"description": "",
"data": {"result": [{"value": 1.0}]}, # Missing timestamp item
},
"thresholds": None,
}
response = self.client.post(self.endpoint, json=parameters)
self.assertEqual(response.status_code, 422)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ def setUp(self):
db_client.get_feature_from_db(dataset="regions", feature_id="12")
)
self.layer = get_layer_fixture("building_area")
self.indicator = BuildingCompleteness(
feature=self.feature, layer=self.layer, thresholds=None
)
self.indicator = BuildingCompleteness(feature=self.feature, layer=self.layer)

@mock.patch("ohsome_quality_analyst.raster.client.get_config_value")
@oqt_vcr.use_cassette()
Expand Down
5 changes: 1 addition & 4 deletions workers/tests/integrationtests/test_indicator_currentness.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ def test(self):
indicator = Currentness(
feature=feature,
layer=get_layer_fixture("major_roads_count"),
thresholds=None,
)
self.assertIsNotNone(indicator.attribution())

Expand All @@ -52,9 +51,7 @@ def test_no_amenities(self):
with open(infile, "r") as f:
feature = geojson.load(f)

indicator = Currentness(
feature=feature, layer=get_layer_fixture("amenities"), thresholds=None
)
indicator = Currentness(feature=feature, layer=get_layer_fixture("amenities"))
asyncio.run(indicator.preprocess())
self.assertEqual(indicator.element_count, 0)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@ def setUp(self):
db_client.get_feature_from_db(dataset="regions", feature_id="3")
)
layer = get_layer_fixture("building_count")
self.indicator = GhsPopComparisonBuildings(
feature=feature, layer=layer, thresholds=None
)
self.indicator = GhsPopComparisonBuildings(feature=feature, layer=layer)

@oqt_vcr.use_cassette()
def test(self):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ def setUp(self):
self.indicator = GhsPopComparisonRoads(
feature=feature,
layer=get_layer_fixture("major_roads_length"),
thresholds=None,
)

@oqt_vcr.use_cassette()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ def setUp(self):
@oqt_vcr.use_cassette()
def test(self):
# Heidelberg
indicator = MappingSaturation(
layer=self.layer, feature=self.feature, thresholds=None
)
indicator = MappingSaturation(layer=self.layer, feature=self.feature)
self.assertIsNotNone(indicator.attribution())

asyncio.run(indicator.preprocess())
Expand Down Expand Up @@ -50,11 +48,7 @@ def test(self):

@oqt_vcr.use_cassette()
def test_as_feature(self):
indicator = MappingSaturation(
layer=self.layer,
feature=self.feature,
thresholds=None,
)
indicator = MappingSaturation(layer=self.layer, feature=self.feature)
asyncio.run(indicator.preprocess())
indicator.calculate()

Expand Down Expand Up @@ -97,7 +91,6 @@ def test_immutable_attribute(self):
indicator = MappingSaturation(
layer=get_layer_fixture("building_count"),
feature=feature,
thresholds=None,
)
asyncio.run(indicator.preprocess())
indicator.calculate()
Expand Down
2 changes: 1 addition & 1 deletion workers/tests/integrationtests/test_indicator_minimal.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ class TestIndicatorMinimal(unittest.TestCase):
def setUp(self):
feature = get_geojson_fixture("heidelberg-altstadt-feature.geojson")
layer = get_layer_fixture("minimal")
self.indicator = Minimal(feature=feature, layer=layer, thresholds=None)
self.indicator = Minimal(feature=feature, layer=layer)

@oqt_vcr.use_cassette()
def test(self):
Expand Down
4 changes: 1 addition & 3 deletions workers/tests/integrationtests/test_indicator_poi_density.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ def setUp(self):
)
with open(infile, "r") as f:
feature = geojson.load(f)
self.indicator = PoiDensity(
feature=feature, layer=get_layer_fixture("poi"), thresholds=None
)
self.indicator = PoiDensity(feature=feature, layer=get_layer_fixture("poi"))

@oqt_vcr.use_cassette()
def test(self):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ def test(self):
indicator = TagsRatio(
feature=self.feature,
layer=get_layer_fixture("jrc_health_count"),
thresholds=None,
)
self.assertIsNotNone(indicator.attribution())

Expand Down
1 change: 0 additions & 1 deletion workers/tests/integrationtests/test_oqt.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,6 @@ def test_create_indicator_as_geojson_size_limit_layer_data(self):
]
},
},
thresholds=None,
)
asyncio.run(oqt.create_indicator_as_geojson(parameters, size_restriction=True))

Expand Down