From 59d59f2a64f6099bcb9ba9d3406bd75ed71143e6 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 12:11:27 +0200 Subject: [PATCH 1/9] Remove Python 2 tests, use Debian buster image --- .circleci/config.yml | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1387007549a..c9336324d84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,37 +87,9 @@ defaults: image: redis:3 name: ckan-redis jobs: - test-python-2: - docker: - - image: python:2-stretch - <<: *ckan_env - - <<: *pg_image - - <<: *redis_image - - parallelism: 4 - - steps: - - checkout - - - <<: *install_deps - - run: | - # Python Dependencies - pip install -r requirement-setuptools.txt - pip install -r requirements-py2.txt - pip install -r dev-requirements.txt - python setup.py develop - pip check - - <<: *init_environment - - <<: *run_tests - - store_test_results: - path: ~/junit - - <<: *start_test_server - - <<: *run_front_tests - - store_artifacts: - path: ~/project/cypress/screenshots test-python-3: docker: - - image: python:3-stretch + - image: python:3-buster <<: *ckan_env - <<: *pg_image - <<: *redis_image @@ -150,5 +122,4 @@ workflows: version: 2 build_and_test: jobs: - - test-python-2 - test-python-3 From f2cc493e932b45799c3100a242e9002c053a82b4 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 12:14:54 +0200 Subject: [PATCH 2/9] Update version for 2.9.9 --- CHANGELOG.rst | 2 ++ ckan/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e23e048230a..7f649e949c0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,8 @@ Bugfixes - `CVE-2023-32321 `_: fix potential path traversal, remote code execution, information disclosure and DOS vulnerabilities via crafted resource ids. +- Names are now quoted in From and To addresses in emails, meaning that site titles with + commas no longer break email clients. (`#7508 `_) Migration notes --------------- diff --git a/ckan/__init__.py b/ckan/__init__.py index 968c1567d94..6411ce74d89 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,6 +1,6 @@ # encoding: utf-8 -__version__ = '2.9.9b' +__version__ = '2.9.9' __description__ = 'CKAN Software' __long_description__ = \ From da0a7a46b48183fa6e2df0824cab97dc19bf8170 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 12:17:04 +0200 Subject: [PATCH 3/9] Fix openjdk package version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9336324d84..fc77f9ddb3a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ defaults: npm install ;; esac - apt install -y postgresql-client solr-jetty openjdk-8-jdk + apt install -y postgresql-client solr-jetty openjdk-11-jdk run_tests: &run_tests # Tests Backend, split across containers by segments run: | From 00ff21b912c04c114484d74b67d4426166bfd478 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 12:28:59 +0200 Subject: [PATCH 4/9] Use py 3.9 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fc77f9ddb3a..592de0f0b9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ defaults: jobs: test-python-3: docker: - - image: python:3-buster + - image: python:3.9-buster <<: *ckan_env - <<: *pg_image - <<: *redis_image From ceeac6d68066200b06ec50b20dc417378daa2e1e Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 12:49:34 +0200 Subject: [PATCH 5/9] Use dockerized solr in tests --- .circleci/config.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 592de0f0b9d..7c516beceb0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,6 @@ version: 2 defaults: init_environemnt: &init_environment run: | - # SOLR config - cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml - service jetty9 restart || true # erroring out but does seem to work - # Database Creation psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;" createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB} @@ -29,7 +25,7 @@ defaults: npm install ;; esac - apt install -y postgresql-client solr-jetty openjdk-11-jdk + apt install -y postgresql-client run_tests: &run_tests # Tests Backend, split across containers by segments run: | @@ -86,6 +82,11 @@ defaults: redis_image: &redis_image image: redis:3 name: ckan-redis + + solr_image: &solr_image + image: ckan/ckan-solr:2.9 + name: ckan-solr + jobs: test-python-3: docker: @@ -93,6 +94,7 @@ jobs: <<: *ckan_env - <<: *pg_image - <<: *redis_image + - <<: *solr_image parallelism: 4 From b73d1b29a9099a78c79909f7cd88c91eaee8d352 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 12:45:07 +0200 Subject: [PATCH 6/9] Don't use Resource factories when creating new Datasets The Resource factory creates an actual Resource model linked to an automatically created dataset. If trying to create a Dataset factory passing a resource one, it will fail as the id already exists --- ckan/tests/logic/auth/test_delete.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/tests/logic/auth/test_delete.py b/ckan/tests/logic/auth/test_delete.py index 1226c1c80e9..52b110d526e 100644 --- a/ckan/tests/logic/auth/test_delete.py +++ b/ckan/tests/logic/auth/test_delete.py @@ -26,7 +26,7 @@ def test_no_org_user_cant_delete(self): user = factories.User() org = factories.Organization() dataset = factories.Dataset( - owner_org=org["id"], resources=[factories.Resource()] + owner_org=org["id"], resources=[{"url": "https://example.com/data.csv"}] ) response = auth_delete.resource_delete( @@ -41,7 +41,7 @@ def test_org_user_can_delete(self): org_users = [{"name": user["name"], "capacity": "editor"}] org = factories.Organization(users=org_users) dataset = factories.Dataset( - owner_org=org["id"], resources=[factories.Resource()], user=user + owner_org=org["id"], resources=[{"url": "https://example.com/data.csv"}], user=user ) response = auth_delete.resource_delete( @@ -67,7 +67,7 @@ def test_no_org_user_cant_delete_2(self): user = factories.User() org = factories.Organization() dataset = factories.Dataset( - owner_org=org["id"], resources=[factories.Resource()] + owner_org=org["id"], resources=[{"url": "https://example.com/data.csv"}] ) resource_view = factories.ResourceView( @@ -88,7 +88,7 @@ def test_org_user_can_delete_2(self): org_users = [{"name": user["name"], "capacity": "editor"}] org = factories.Organization(users=org_users) dataset = factories.Dataset( - owner_org=org["id"], resources=[factories.Resource()], user=user + owner_org=org["id"], resources=[{"url": "https://example.com/data.csv"}], user=user ) resource_view = factories.ResourceView( From 90f7ab8716f07e52a6b12fa4dcfc11c32c240194 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 12:55:46 +0200 Subject: [PATCH 7/9] Use proper solr url --- test-core-circle-ci.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-core-circle-ci.ini b/test-core-circle-ci.ini index a558853e415..70c9d0f7801 100644 --- a/test-core-circle-ci.ini +++ b/test-core-circle-ci.ini @@ -11,7 +11,7 @@ ckan.redis.url = redis://ckan-redis:6379/1 sqlalchemy.url = postgresql://ckan_default:pass@ckan-postgres/ckan_test -solr_url = http://localhost:8080/solr +solr_url = http://ckan-solr:8983/solr/ckan [loggers] keys = root, ckan, sqlalchemy From 535cd70362befe4a30dc44475a8bf64a78d3f493 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 13:12:58 +0200 Subject: [PATCH 8/9] pep8 --- ckan/lib/uploader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index e8e68e58ca0..ec4a6dfd0b1 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -294,7 +294,8 @@ def get_directory(self, id): real_storage = os.path.realpath(self.storage_path) directory = os.path.join(real_storage, id[0:3], id[3:6]) if directory != os.path.realpath(directory): - raise logic.ValidationError({'upload': ['Invalid storage directory']}) + raise logic.ValidationError( + {'upload': ['Invalid storage directory']}) return directory def get_path(self, id): From 97945fb898011be9b5c0c857f59fef090a5864e5 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 24 May 2023 13:20:36 +0200 Subject: [PATCH 9/9] Fix another resource factory --- ckanext/datapusher/tests/test_interfaces.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckanext/datapusher/tests/test_interfaces.py b/ckanext/datapusher/tests/test_interfaces.py index cc39a5c65f6..c8fa2d8b02d 100644 --- a/ckanext/datapusher/tests/test_interfaces.py +++ b/ckanext/datapusher/tests/test_interfaces.py @@ -41,8 +41,10 @@ def setup_class(self, clean_db, test_request_context): if not tests.is_datastore_supported(): pytest.skip("Datastore not supported") - resource = factories.Resource(url_type="datastore") - self.dataset = factories.Dataset(resources=[resource]) + self.dataset = factories.Dataset( + resources=[ + {"url_type": "datastore"} + ]) with test_request_context(): self.sysadmin_user = factories.User( name="testsysadmin", sysadmin=True