diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100755 index 0000000..f0dda16 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,25 @@ +node { + stage 'Retrieve sources' + checkout([ + $class: 'GitSCM', branches: [[name: 'refs/heads/'+env.BRANCH_NAME]], + extensions: [[$class: 'CloneOption', noTags: false, shallow: false, depth: 0, reference: '']], + userRemoteConfigs: scm.userRemoteConfigs, + ]) + + stage 'Clean' + sh 'rm -rf ./ci' + sh 'mkdir -p ./ci' + + stage 'Compute version name' + sh 'scripts/ciBuildVersion.sh ${BRANCH_NAME}' + + stage 'Download and cache dependencies' + sh 'scripts/ciCreateDependencyImage.sh' + + stage 'Test' + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh 'scripts/ciTest.sh' + } + stage 'Publish test' + step([$class: 'JUnitResultArchiver', testResults: '**/ci/test-reports/*.xml']) +} diff --git a/pims_plugin_format_bioformats/config.py b/pims_plugin_format_bioformats/config.py index bf39a73..e098c9d 100644 --- a/pims_plugin_format_bioformats/config.py +++ b/pims_plugin_format_bioformats/config.py @@ -19,7 +19,7 @@ class Settings(BaseSettings): - bioformats_host = "localhost" + bioformats_host = "bioformat" bioformats_port = 4321 bioformats_metadata_timeout = 15 bioformats_conversion_timeout = 200 * 60 diff --git a/pims_plugin_format_bioformats/czi.py b/pims_plugin_format_bioformats/czi.py index 1f6aaf0..c2e04b7 100644 --- a/pims_plugin_format_bioformats/czi.py +++ b/pims_plugin_format_bioformats/czi.py @@ -48,7 +48,7 @@ class CZIFormat(AbstractFormat): """ checker_class = CZIChecker parser_class = BioFormatsParser - reader_class = BioFormatsReader + reader_class = None histogram_reader_class = DefaultHistogramReader convertor_class = BioFormatsSpatialConvertor diff --git a/pims_plugin_format_bioformats/lif.py b/pims_plugin_format_bioformats/lif.py index 76fa100..e730213 100644 --- a/pims_plugin_format_bioformats/lif.py +++ b/pims_plugin_format_bioformats/lif.py @@ -47,7 +47,7 @@ class LIFFormat(AbstractFormat): """ checker_class = LIFChecker parser_class = BioFormatsParser - reader_class = BioFormatsReader + reader_class = None histogram_reader_class = DefaultHistogramReader convertor_class = BioFormatsSpatialConvertor diff --git a/pims_plugin_format_bioformats/nd2.py b/pims_plugin_format_bioformats/nd2.py index a5da394..e519af6 100644 --- a/pims_plugin_format_bioformats/nd2.py +++ b/pims_plugin_format_bioformats/nd2.py @@ -54,7 +54,7 @@ class ND2Format(AbstractFormat): """ checker_class = ND2Checker parser_class = BioFormatsParser - reader_class = BioFormatsReader + reader_class = None histogram_reader_class = DefaultHistogramReader convertor_class = BioFormatsSpatialConvertor diff --git a/pims_plugin_format_bioformats/utils/engine.py b/pims_plugin_format_bioformats/utils/engine.py index 6978971..81a422c 100644 --- a/pims_plugin_format_bioformats/utils/engine.py +++ b/pims_plugin_format_bioformats/utils/engine.py @@ -12,7 +12,7 @@ # * See the License for the specific language governing permissions and # * limitations under the License. from __future__ import annotations - +import os import json import logging import select @@ -87,7 +87,7 @@ def _data_available(): parsed_response = json.loads(response) if not silent_fail and 'error' in parsed_response: - raise ValueError # TODO: better error + raise ValueError(parsed_response['error']) # TODO: better error return parsed_response except InterruptedError as e: @@ -309,7 +309,9 @@ def need_pyramid(self) -> bool: return not (imd.width <= self.TILE_SIZE or imd.height <= self.TILE_SIZE) def convert(self, dest_path: Path) -> bool: - intermediate_path = dest_path.with_stem("intermediate").with_suffix(".tmp") + from pims.files.file import Path + intermediate_path = Path(os.path.join(os.path.split(dest_path)[0],"intermediate.tmp")) + #intermediate_path = dest_path.with_stem("intermediate").with_suffix(".tmp") message = { "action": "convert", "legacyMode": False, diff --git a/scripts/README_BUILD.md b/scripts/README_BUILD.md new file mode 100644 index 0000000..11eaf04 --- /dev/null +++ b/scripts/README_BUILD.md @@ -0,0 +1,29 @@ +# PIMS Isyntax plugin - Build & Continous integration + +## Jenkins file + +A `Jenkinsfile` is located at the root of the directory. +Build steps: +* Clean `./ci` directory ; In this directory we will store all temp data for the build. +* Get the current version (see Versionning section) +* Create a Docker image with all dependencies +* Run tests + +The `scripts/ciBuildLocal.sh` contains same steps as Jenkinsfile but can be run without Jenkins. + +## Tests + +Tests are run with pytest. +The test report is extracted as a XML file in `ci/test-reports` + +## Final build + +No final build, as the source code on the repo is enough. + +## Versioning + +The release version is currently not supported. +Multiple possibilities: +* Manually manage version in `__version__.py` file +* Each build with an official release (x.y.z) could automatically modify the source code on the repository to update the version. +* Each build export a zip file of the code with the updated version that could be stored somewhere. \ No newline at end of file diff --git a/scripts/ciBuildLocal.sh b/scripts/ciBuildLocal.sh new file mode 100755 index 0000000..25bf1d7 --- /dev/null +++ b/scripts/ciBuildLocal.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -o xtrace +set -o errexit +set -a + +rm -rf ./ci +mkdir ./ci + +./scripts/ciBuildVersion.sh + +./scripts/ciCreateDependencyImage.sh + +./scripts/ciTest.sh + + +rm -rf ./ci +mkdir ./ci diff --git a/scripts/ciBuildVersion.sh b/scripts/ciBuildVersion.sh new file mode 100755 index 0000000..bab0643 --- /dev/null +++ b/scripts/ciBuildVersion.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +srcRoot=$(git rev-parse --show-toplevel) +cd $srcRoot + +# get version number from git +gitLongTag=$(git describe --long --dirty --tags) +# get the branch name from first arg or from git +branchName=${1:-$(git rev-parse --abbrev-ref HEAD)} + +# check if tag is an official release (v1.2.3) + no other commit behind (or dirty) +if [[ $gitLongTag =~ v[0-9]+.[0-9]+.[0-9]+-0-[0-9a-g]{8,9}$ ]]; then + versionNumber=$(echo $gitLongTag | sed -r "s/v([0-9]+\.[0-9]+\.[0-9]+)-[0-9]+-.+/\1/") +else + echo "WARNING: invalid tag for an official release $gitLongTag" + versionNumber=$branchName-$(date "+%Y%m%d%H%M%S")-SNAPSHOT +fi + +echo $versionNumber > ./ci/version \ No newline at end of file diff --git a/scripts/ciCreateDependencyImage.sh b/scripts/ciCreateDependencyImage.sh new file mode 100755 index 0000000..4fde74d --- /dev/null +++ b/scripts/ciCreateDependencyImage.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +echo "************************************** Create dependency image ******************************************" + +file='./ci/version' +VERSION_NUMBER=$(<"$file") + +echo "Launch Create dependency image for $VERSION_NUMBER" + +git clone --branch master --depth 1 https://github.com/cytomine/pims ./ci/app + +mkdir -p ./ci/app/plugins/pims-plugin-format-bioformats/ +cp -r ./pims_plugin_format_bioformats ./ci/app/plugins/pims-plugin-format-bioformats/ +#cp -r ./pims_plugin_format_bioformats.egg-info ./ci/app/plugins/pims-plugin-format-bioformats/ +cp -r ./tests ./ci/app/plugins/pims-plugin-format-bioformats/ +cp ./setup.py ./ci/app/plugins/pims-plugin-format-bioformats/ + +#git clone https://github.com/cytomine/pims-plugin-format-openslide ./ci/app/plugins/pims-plugin-format-openslide + +docker build --rm -f scripts/docker/Dockerfile-dependencies -t cytomine/pims-plugin-format-bioformats-dependencies:v$VERSION_NUMBER . diff --git a/scripts/ciTest.sh b/scripts/ciTest.sh new file mode 100755 index 0000000..8de4087 --- /dev/null +++ b/scripts/ciTest.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +echo "************************************** Launch tests ******************************************" + +file='./ci/version' +VERSION_NUMBER=$(<"$file") + +echo "Launch tests for $VERSION_NUMBER" +mkdir "$PWD"/ci/test-reports +touch "$PWD"/ci/test-reports/pytest_unit.xml +docker build --rm -f scripts/docker/Dockerfile-test --build-arg VERSION_NUMBER=$VERSION_NUMBER -t cytomine/pims-plugin-format-bioformats-test . + +containerIdBioformat=$(docker create --name bioformat -v /data/images:/data/images -v /data/pims:/data/pims -e BIOFORMAT_PORT=4321 --restart=unless-stopped cytomine/bioformat:v3.1.0 ) +docker start $containerIdBioformat + +containerId=$(docker create --link bioformat:bioformat -v /data/pims:/data/pims -v "$PWD"/ci/test-reports:/app/ci/test-reports -v /tmp/uploaded:/tmp/uploaded -v /tmp/test-pims:/tmp/test-pims cytomine/pims-plugin-format-bioformats-test ) + +docker start -ai $containerId +docker rm $containerId +docker stop $containerIdBioformat +docker rm $containerIdBioformat diff --git a/scripts/docker/Dockerfile-dependencies b/scripts/docker/Dockerfile-dependencies new file mode 100644 index 0000000..b298b8b --- /dev/null +++ b/scripts/docker/Dockerfile-dependencies @@ -0,0 +1,146 @@ +FROM ubuntu:20.04 + +ENV LANG C.UTF-8 +ENV DEBIAN_FRONTEND noninteractive + +ARG PY_VERSION=3.8 + +RUN apt-get -y update && apt-get -y install --no-install-recommends --no-install-suggests \ + `# Essentials` \ + automake \ + build-essential \ + ca-certificates \ + cmake \ + git \ + gcc \ + net-tools \ + python${PY_VERSION} \ + python${PY_VERSION}-dev \ + python${PY_VERSION}-distutils \ + wget \ + software-properties-common \ + `# Vips dependencies` \ + pkg-config \ + glib2.0-dev \ + libexpat1-dev \ + libtiff5-dev \ + libjpeg-turbo8 \ + libgsf-1-dev \ + libexif-dev \ + libvips-dev \ + orc-0.4-dev \ + libwebp-dev \ + liblcms2-dev \ + libpng-dev \ + gobject-introspection \ + `# Other tools` \ + libimage-exiftool-perl + +RUN cd /usr/bin && \ + ln -s python${PY_VERSION} python + +# Official pip install: https://pip.pypa.io/en/stable/installation/#get-pip-py +RUN cd /tmp && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py && \ + rm -rf get-pip.py + +# openjpeg 2.4 is required by vips (J2000 support) +ARG OPENJPEG_VERSION=2.4.0 +ARG OPENJPEG_URL=https://github.com/uclouvain/openjpeg/archive +RUN cd /usr/local/src && \ + wget ${OPENJPEG_URL}/v${OPENJPEG_VERSION}/openjpeg-${OPENJPEG_VERSION}.tar.gz && \ + tar -zxvf openjpeg-${OPENJPEG_VERSION}.tar.gz && \ + rm -rf openjpeg-${OPENJPEG_VERSION}.tar.gz && \ + cd openjpeg-${OPENJPEG_VERSION} && \ + mkdir build && cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_STATIC_LIBS=ON .. && \ + make && \ + make install && \ + make clean && \ + ldconfig + +# Download plugins +WORKDIR /app +COPY ./ci/app/docker/plugins.py /app/plugins.py + +ARG PLUGIN_CSV +# ="enabled,name,git_url,git_branch\n" +ENV PLUGIN_INSTALL_PATH /app/plugins +RUN python3 plugins.py \ + --plugin_csv ${PLUGIN_CSV} \ + --install_path ${PLUGIN_INSTALL_PATH} \ + --method download + +# Run before_vips() from plugins prerequisites +RUN python3 plugins.py \ + --plugin_csv ${PLUGIN_CSV} \ + --install_path ${PLUGIN_INSTALL_PATH} \ + --method dependencies_before_vips + +# vips +ARG VIPS_VERSION=8.11.2 +ARG VIPS_URL=https://github.com/libvips/libvips/releases/download +RUN cd /usr/local/src && \ + wget ${VIPS_URL}/v${VIPS_VERSION}/vips-${VIPS_VERSION}.tar.gz && \ + tar -zxvf vips-${VIPS_VERSION}.tar.gz && \ + rm -rf vips-${VIPS_VERSION}.tar.gz && \ + cd vips-${VIPS_VERSION} && \ + ./configure && \ + make V=0 && \ + make install && \ + ldconfig + +# Run before_python() from plugins prerequisites +RUN python3 plugins.py \ + --plugin_csv ${PLUGIN_CSV} \ + --install_path ${PLUGIN_INSTALL_PATH} \ + --method dependencies_before_python + + +# Cleaning. Cannot be done before as plugin prerequisites could use apt-get. +RUN rm -rf /var/lib/apt/lists/* + +# Install python requirements +COPY ./ci/app/requirements.txt /app/requirements.txt + +ARG GUNICORN_VERSION=20.1.0 +RUN pip3 install gunicorn==${GUNICORN_VERSION} && \ + pip3 install -r requirements.txt && \ + python3 plugins.py \ + --plugin_csv ${PLUGIN_CSV} \ + --install_path ${PLUGIN_INSTALL_PATH} \ + --method install + +#Install plugins +# COPY ./ci/app/plugins/pims-plugin-format-openslide /app/plugins/pims-plugin-format-openslide/ +COPY ./ci/app/plugins/pims-plugin-format-bioformats /app/plugins/pims-plugin-format-bioformats/ +# RUN pip3 install /app/plugins/pims-plugin-format-openslide +RUN pip3 install /app/plugins/pims-plugin-format-bioformats + +# Prestart configuration +RUN touch /tmp/addHosts.sh +COPY ./ci/app/docker/prestart.sh /app/prestart.sh +RUN chmod +x /app/prestart.sh + +# Add default config +COPY ./ci/app/pims-config.env /app/pims-config.env +COPY ./ci/app/logging-prod.yml /app/logging-prod.yml +COPY ./ci/app/docker/gunicorn_conf.py /app/gunicorn_conf.py + +COPY ./ci/app/docker/start.sh /start.sh +RUN chmod +x /start.sh + +COPY ./ci/app/docker/start-reload.sh /start-reload.sh +RUN chmod +x /start-reload.sh + +ENV PYTHONPATH=/app + +# Add app +COPY ./ci/app/pims /app/pims +COPY ./ci/app/setup.py /app +ENV MODULE_NAME="pims.application" + +ENV PORT=5000 +EXPOSE ${PORT} +CMD ["/start.sh"] diff --git a/scripts/docker/Dockerfile-final b/scripts/docker/Dockerfile-final new file mode 100644 index 0000000..68aff28 --- /dev/null +++ b/scripts/docker/Dockerfile-final @@ -0,0 +1,7 @@ +ARG VERSION_NUMBER +FROM cytomine/pims-plugin-format-bioformats-dependencies:v$VERSION_NUMBER + +ENV MODULE_NAME="pims.application" +ENV PORT=5000 +EXPOSE ${PORT} +CMD ["/start.sh"] diff --git a/scripts/docker/Dockerfile-test b/scripts/docker/Dockerfile-test new file mode 100644 index 0000000..9c910fc --- /dev/null +++ b/scripts/docker/Dockerfile-test @@ -0,0 +1,14 @@ +ARG VERSION_NUMBER +FROM cytomine/pims-plugin-format-bioformats-dependencies:v$VERSION_NUMBER + +RUN pip3 install pytest +RUN pip3 install -e /app + + +COPY ./ci/app/tests /app/tests + +RUN ls /app/ + +RUN ls /app/tests + +CMD pytest -rP /app/plugins/pims-plugin-format-bioformats/tests --junit-xml=ci/test-reports/pytest_unit.xml diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..73b3364 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,92 @@ +# * Copyright (c) 2020-2021. Authors: see NOTICE file. +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +import os +import shutil +from contextlib import contextmanager +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from pims import config + +os.environ['CONFIG_FILE'] = "./pims-config.env" + +def get_test_root(): + return get_settings().root + +@pytest.fixture +def root(): + return get_test_root() + + +def get_settings(): + return config.Settings( + _env_file=os.getenv("CONFIG_FILE") + ) +""" +def get_pims_root(): + print(get_settings()) + return get_settings().root_pims + +@pytest.fixture +def pims_root(): + return get_pims_root() +""" +@pytest.fixture +def settings(): + return get_settings() + + +@pytest.fixture +def app(): + from pims import application as main + + main.app.dependency_overrides[config.get_settings] = get_settings + return main.app + + +@pytest.fixture +def client(app): + return TestClient(app) + +@pytest.fixture +def image_path_czi(): + path = f"{get_test_root()}/upload_test_bioformats_czi" + filename = "Plate1-Blue-A-12-Scene-3-P3-F2-03.czi" + return [path, filename] + +@pytest.fixture +def image_path_nd2(): + path = f"{get_test_root()}/upload_test_bioformats_nd2" + filename = "BF007.nd2" + return [path, filename] + +@contextmanager +def not_raises(expected_exc): + try: + yield + + except expected_exc as err: + raise AssertionError( + "Did raise exception {0} when it should not!".format( + repr(expected_exc) + ) + ) + + except Exception as err: + raise AssertionError( + "An unexpected exception {0} raised.".format(repr(err)) + ) diff --git a/tests/pims-config.env b/tests/pims-config.env new file mode 100755 index 0000000..4461f76 --- /dev/null +++ b/tests/pims-config.env @@ -0,0 +1,8 @@ +ROOT="/tmp/test-pims" +PENDING_PATH="/tmp/uploaded" +DEFAULT_IMAGE_SIZE_SAFETY_MODE="SAFE_REJECT" +DEFAULT_ANNOTATION_ORIGIN="LEFT_TOP" +OUTPUT_SIZE_LIMIT="10000" +CYTOMINE_PUBLIC_KEY="123" +CYTOMINE_PRIVATE_KEY="456" +PIMS_URL="http://localhost-ims" diff --git a/tests/test_czi.py b/tests/test_czi.py new file mode 100644 index 0000000..a816937 --- /dev/null +++ b/tests/test_czi.py @@ -0,0 +1,143 @@ +from PIL import Image +import os, io, json +import urllib.request +from fastapi import APIRouter +from pims.formats import FORMATS +from pims.importer.importer import FileImporter + +from pims.files.file import (ORIGINAL_STEM, Path, SPATIAL_STEM, HISTOGRAM_STEM) + +from pims.api.utils.models import HistogramType +from pims.processing.histograms.utils import build_histogram_file +from pims.formats.utils.factories import FormatFactory +from pims.utils.dtypes import dtype_to_bits +import pytest +import subprocess +from subprocess import Popen, PIPE, STDOUT + +def get_image(root, path, filename): + filepath = os.path.join(path, filename) + # If image does not exist locally -> download image + + if not os.path.exists("/tmp/images"): + os.mkdir("/tmp/images") + + if not os.path.exists(root): + os.mkdir(root) + + if not os.path.exists(f"/tmp/images/{filename}"): + try: + url = f"https://downloads.openmicroscopy.org/images/Zeiss-CZI/idr0011/Plate1-Blue-A_TS-Stinger/{filename}" + urllib.request.urlretrieve(url, f"/tmp/images/{filename}") + except Exception as e: + print("Could not download image") + print(e) + if not os.path.exists(filepath): + image_path = f"/tmp/images/{filename}" + pims_root = root + importer_path = f"/app/pims/importer/import_local_images.py" # pims folder should be in root folder + import_img=subprocess.run(["python3", importer_path, "--path", image_path], stdout=subprocess.PIPE) + + subdirs = os.listdir(pims_root) + for subdir in subdirs: + if "upload-" in str(subdir): + subsubdirs = os.listdir(os.path.join(root, subdir)) + for subsubdir in subsubdirs: + if filename in str(subsubdir): + upload_dir = os.path.join(root, str(subdir)) + break + if os.path.exists(path): + os.unlink(path) # if the folder upload_test_bioformats_czi already exists the symlink won't work + os.symlink(upload_dir, path) + +def test_bioformats_czi_exists(image_path_czi, settings): + # Test if the file exists, either locally either with the OAC + path, filename = image_path_czi + get_image(settings.root, path, filename) + assert os.path.exists(os.path.join(path,filename)) == True + +def test_format_exists(client): + response = client.get(f'/formats') + assert "CZI" in json.dumps(response.json()) + +def test_bioformats_czi_info(client, image_path_czi): + path, filename = image_path_czi + response = client.get(f'/image/upload_test_bioformats_czi/{filename}/info') + assert response.status_code == 200 + assert "czi" in response.json()['image']['original_format'].lower() + + assert response.json()['image']['width'] == 672 + assert response.json()['image']['height'] == 512 + + +def test_bioformats_czi_norm_tile(client, image_path_czi): + _, filename = image_path_czi + response = client.get(f"/image/upload_test_bioformats_czi/{filename}/normalized-tile/level/0/ti/0", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + img_response = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = img_response.size + assert width_resp == 256 + assert height_resp == 256 + +def test_bioformats_czi_thumb(client, image_path_czi): + _, filename = image_path_czi + response = client.get(f"/image/upload_test_bioformats_czi/{filename}/thumb", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + assert max(width_resp, height_resp) == 256 + + +@pytest.mark.skip(reason='There is no associated macroimage') +def test_bioformats_czi_macro(client, image_path_czi): + path, filename = image_path_czi + response = client.get(f"/image/upload_test_bioformats_czi/{filename}/associated/macro", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + assert width_resp == 256 or height_resp == 256 + +@pytest.mark.skip(reason='There is no associated label image') +def test_bioformats_czi_label(client, image_path_czi): + path, filename = image_path_czi + response = client.get(f"/image/upload_test_bioformats_czi/{filename}/associated/label", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + assert width_resp == 256 or height_resp == 256 + +def test_bioformats_czi_resized(client, image_path_czi): + _, filename = image_path_czi + response = client.get(f"/image/upload_test_bioformats_czi/{filename}/resized", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + #assert width_resp == 256 or height_resp == 256 + assert max(width_resp, height_resp) == 256 + +def test_bioformats_czi_mask(client, image_path_czi): + _, filename = image_path_czi + response = client.post(f"/image/upload_test_bioformats_czi/{filename}/annotation/mask", headers={"accept": "image/jpeg"}, json={"annotations":[{"geometry": "POINT(10 10)"}], "height":50, "width":50}) + assert response.status_code == 200 + +def test_bioformats_czi_crop(client, image_path_czi): + _, filename = image_path_czi + response = client.post(f"/image/upload_test_bioformats_czi/{filename}/annotation/crop", headers={"accept": "image/jpeg"}, json={"annotations":[{"geometry": "POINT(10 10)"}], "height":50, "width":50}) + assert response.status_code == 200 + +@pytest.mark.skip(reason="Does not return the correct response code") +def test_tiff_crop_null_annot(client, image_path_czi): + _,filename = image_path_czi + response = client.post(f"/image/upload_test_bioformats_czi/{filename}/annotation/crop", headers={"accept": "image/jpeg"}, json={"annotations": [], "height":50, "width":50}) + assert response.status_code == 400 + + +def test_bioformats_czi_histogram_perimage(client, image_path_czi): + _, filename = image_path_czi + response = client.get(f"/image/upload_test_bioformats_czi/{filename}/histogram/per-image", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 diff --git a/tests/test_nd2.py b/tests/test_nd2.py new file mode 100644 index 0000000..a40d181 --- /dev/null +++ b/tests/test_nd2.py @@ -0,0 +1,146 @@ +from PIL import Image +import os, io, json +import urllib.request +from fastapi import APIRouter +from pims.formats import FORMATS +from pims.importer.importer import FileImporter + +from pims.files.file import (ORIGINAL_STEM, Path, SPATIAL_STEM, HISTOGRAM_STEM) + +from pims.api.utils.models import HistogramType +from pims.processing.histograms.utils import build_histogram_file +from pims.formats.utils.factories import FormatFactory +from pims.utils.dtypes import dtype_to_bits +#from pims.tests.utils.formats import info_test, thumb_test, resized_test, mask_test, crop_test, crop_null_annot_test, histogram_perimage_test + +import pytest +import subprocess +from subprocess import Popen, PIPE, STDOUT + +def get_image(root, path, filename): + filepath = os.path.join(path, "/", filename) + # If image does not exist locally -> download image + + if not os.path.exists("/tmp/images"): + os.mkdir("/tmp/images") + + if not os.path.exists(root): + os.mkdir(root) + + if not os.path.exists(f"/tmp/images/{filename}"): + try: + url = f"https://downloads.openmicroscopy.org/images/ND2/maxime/{filename}" + urllib.request.urlretrieve(url, f"/tmp/images/{filename}") + except Exception as e: + print("Could not download image") + print(e) + + if not os.path.exists(filepath): + image_path = f"/tmp/images/{filename}" + pims_root = root + importer_path = f"/app/pims/importer/import_local_images.py" # pims folder should be in root folder + import_img=subprocess.run(["python3", importer_path, "--path", image_path]) + + subdirs = os.listdir(pims_root) + for subdir in subdirs: + if "upload-" in str(subdir): + subsubdirs = os.listdir(os.path.join(root, subdir)) + for subsubdir in subsubdirs: + if ".nd2" in str(subsubdir): + upload_dir = os.path.join(root, str(subdir)) + break + if os.path.exists(path): + os.unlink(path) # if the folder upload_test_bioformats_nd2 already exists the symlink won't work + os.symlink(upload_dir, path) + +def test_bioformats_nd2_exists(image_path_nd2, settings): + # Test if the file exists, either locally either with the OAC + path, filename = image_path_nd2 + get_image(settings.root, path, filename) + assert os.path.exists(os.path.join(path,filename)) == True + +def test_format_exists(client): + response = client.get(f'/formats') + assert "nd2" in json.dumps(response.json()).lower() + +def test_bioformats_nd2_info(client, image_path_nd2): + path, filename = image_path_nd2 + response = client.get(f'/image/upload_test_bioformats_nd2/{filename}/info') + assert response.status_code == 200 + assert "nd2" in response.json()['image']['original_format'].lower() + + assert response.json()['image']['width'] == 164 + assert response.json()['image']['height'] == 156 + + +def test_bioformats_nd2_norm_tile(client, image_path_nd2): + _, filename = image_path_nd2 + response = client.get(f"/image/upload_test_bioformats_nd2/{filename}/normalized-tile/level/0/ti/0", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + img_response = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = img_response.size + assert width_resp == 164 + assert height_resp == 156 + +def test_bioformats_nd2_thumb(client, image_path_nd2): + _, filename = image_path_nd2 + response = client.get(f"/image/upload_test_bioformats_nd2/{filename}/thumb", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + assert max(width_resp, height_resp) == 256 + + +@pytest.mark.skip(reason='There is no associated macroimage') +def test_bioformats_nd2_macro(client, image_path_nd2): + path, filename = image_path_nd2 + response = client.get(f"/image/upload_test_bioformats_nd2/{filename}/associated/macro", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + assert width_resp == 256 or height_resp == 256 + +@pytest.mark.skip(reason='There is no associated label image') +def test_bioformats_nd2_label(client, image_path_nd2): + path, filename = image_path_nd2 + response = client.get(f"/image/upload_test_bioformats_nd2/{filename}/associated/label", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + assert width_resp == 256 or height_resp == 256 + +def test_bioformats_nd2_resized(client, image_path_nd2): + _, filename = image_path_nd2 + response = client.get(f"/image/upload_test_bioformats_nd2/{filename}/resized", headers={"accept": "image/jpeg"}) + assert response.status_code == 200 + + im_resp = Image.open(io.BytesIO(response.content)) + width_resp, height_resp = im_resp.size + #assert width_resp == 256 or height_resp == 256 + assert max(width_resp, height_resp) == 256 + +def test_bioformats_nd2_mask(client, image_path_nd2): + _, filename = image_path_nd2 + response = client.post(f"/image/upload_test_bioformats_nd2/{filename}/annotation/mask", headers={"accept": "image/jpeg"}, json={"annotations":[{"geometry": "POINT(10 10)"}], "height":50, "width":50}) + assert response.status_code == 200 + +def test_bioformats_nd2_crop(client, image_path_nd2): + _, filename = image_path_nd2 + response = client.post(f"/image/upload_test_bioformats_nd2/{filename}/annotation/crop", headers={"accept": "image/jpeg"}, json={"annotations":[{"geometry": "POINT(10 10)"}], "height":50, "width":50}) + assert response.status_code == 200 + +@pytest.mark.skip(reason="Does not return the correct response code") +def test_tiff_crop_null_annot(client, image_path_nd2): + _,filename = image_path_nd2 + response = client.post(f"/image/upload_test_bioformats_nd2/{filename}/annotation/crop", headers={"accept": "image/jpeg"}, json={"annotations": [], "height":50, "width":50}) + assert response.status_code == 400 + + +def test_bioformats_nd2_histogram_perimage(client, image_path_nd2): + _, filename = image_path_nd2 + response = client.get(f"/image/upload_test_bioformats_nd2/{filename}/histogram/per-image", headers={"accept": "image/jpeg"}) + assert response.status_code == 200