BioFormats formats are not readable and conversion is mandatory

Jenkinsfile

@@ -0,0 +1,25 @@
+node {
+    stage 'Retrieve sources'
+    checkout([
+        $class: 'GitSCM',  branches: [[name: 'refs/heads/'+env.BRANCH_NAME]],
+        extensions: [[$class: 'CloneOption', noTags: false, shallow: false, depth: 0, reference: '']],
+        userRemoteConfigs: scm.userRemoteConfigs,
+    ])
+
+    stage 'Clean'
+    sh 'rm -rf ./ci'
+    sh 'mkdir -p ./ci'
+
+    stage 'Compute version name'
+    sh 'scripts/ciBuildVersion.sh ${BRANCH_NAME}'
+
+    stage 'Download and cache dependencies'
+    sh 'scripts/ciCreateDependencyImage.sh'
+
+    stage 'Test'
+    catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+        sh 'scripts/ciTest.sh'
+    }
+    stage 'Publish test'
+    step([$class: 'JUnitResultArchiver', testResults: '**/ci/test-reports/*.xml'])
+}

pims_plugin_format_bioformats/config.py

@@ -19,7 +19,7 @@
 
 
 class Settings(BaseSettings):
-    bioformats_host = "localhost"
+    bioformats_host = "bioformat"
     bioformats_port = 4321
     bioformats_metadata_timeout = 15
     bioformats_conversion_timeout = 200 * 60

pims_plugin_format_bioformats/czi.py

@@ -48,7 +48,7 @@ class CZIFormat(AbstractFormat):
     """
     checker_class = CZIChecker
     parser_class = BioFormatsParser
-    reader_class = BioFormatsReader
+    reader_class = None
     histogram_reader_class = DefaultHistogramReader
     convertor_class = BioFormatsSpatialConvertor
 

pims_plugin_format_bioformats/lif.py

@@ -47,7 +47,7 @@ class LIFFormat(AbstractFormat):
     """
     checker_class = LIFChecker
     parser_class = BioFormatsParser
-    reader_class = BioFormatsReader
+    reader_class = None
     histogram_reader_class = DefaultHistogramReader
     convertor_class = BioFormatsSpatialConvertor
 

pims_plugin_format_bioformats/nd2.py

@@ -54,7 +54,7 @@ class ND2Format(AbstractFormat):
     """
     checker_class = ND2Checker
     parser_class = BioFormatsParser
-    reader_class = BioFormatsReader
+    reader_class = None
     histogram_reader_class = DefaultHistogramReader
     convertor_class = BioFormatsSpatialConvertor
 

pims_plugin_format_bioformats/utils/engine.py

@@ -12,7 +12,7 @@
 #  * See the License for the specific language governing permissions and
 #  * limitations under the License.
 from __future__ import annotations
-
+import os
 import json
 import logging
 import select
@@ -87,7 +87,7 @@ def _data_available():
 
             parsed_response = json.loads(response)
             if not silent_fail and 'error' in parsed_response:
-                raise ValueError  # TODO: better error
+                raise ValueError(parsed_response['error']) # TODO: better error
 
             return parsed_response
     except InterruptedError as e:
@@ -309,7 +309,9 @@ def need_pyramid(self) -> bool:
         return not (imd.width <= self.TILE_SIZE or imd.height <= self.TILE_SIZE)
 
     def convert(self, dest_path: Path) -> bool:
-        intermediate_path = dest_path.with_stem("intermediate").with_suffix(".tmp")
+        from pims.files.file import Path
+        intermediate_path = Path(os.path.join(os.path.split(dest_path)[0],"intermediate.tmp"))
+        #intermediate_path = dest_path.with_stem("intermediate").with_suffix(".tmp")
         message = {
             "action": "convert",
             "legacyMode": False,

scripts/README_BUILD.md

@@ -0,0 +1,29 @@
+# PIMS Isyntax plugin - Build & Continous integration
+
+## Jenkins file
+
+A `Jenkinsfile` is located at the root of the directory.
+Build steps:
+* Clean `./ci` directory ; In this directory we will store all temp data for the build.
+* Get the current version (see Versionning section)
+* Create a Docker image with all dependencies
+* Run tests
+
+The `scripts/ciBuildLocal.sh` contains same steps as Jenkinsfile but can be run without Jenkins.
+
+## Tests
+
+Tests are run with pytest.
+The test report is extracted as a XML file in `ci/test-reports`
+
+## Final build
+
+No final build, as the source code on the repo is enough.
+
+## Versioning
+
+The release version is currently not supported.
+Multiple possibilities:
+* Manually manage version in `__version__.py` file
+* Each build with an official release (x.y.z) could automatically modify the source code on the repository to update the version.
+* Each build export a zip file of the code with the updated version that could be stored somewhere.

scripts/ciBuildLocal.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+set -a
+
+rm -rf ./ci
+mkdir ./ci
+
+./scripts/ciBuildVersion.sh
+
+./scripts/ciCreateDependencyImage.sh
+
+./scripts/ciTest.sh
+
+
+rm -rf ./ci
+mkdir ./ci

scripts/ciBuildVersion.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+srcRoot=$(git rev-parse --show-toplevel)
+cd $srcRoot
+
+# get version number from git
+gitLongTag=$(git describe --long --dirty --tags)
+# get the branch name from first arg or from git
+branchName=${1:-$(git rev-parse --abbrev-ref HEAD)}
+
+# check if tag is an official release (v1.2.3) + no other commit behind (or dirty)
+if [[ $gitLongTag =~ v[0-9]+.[0-9]+.[0-9]+-0-[0-9a-g]{8,9}$ ]]; then
+  versionNumber=$(echo $gitLongTag | sed -r "s/v([0-9]+\.[0-9]+\.[0-9]+)-[0-9]+-.+/\1/")
+else
+  echo "WARNING: invalid tag for an official release $gitLongTag"
+  versionNumber=$branchName-$(date "+%Y%m%d%H%M%S")-SNAPSHOT
+fi
+
+echo $versionNumber > ./ci/version

scripts/ciCreateDependencyImage.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+echo "************************************** Create dependency image ******************************************"
+
+file='./ci/version'
+VERSION_NUMBER=$(<"$file")
+
+echo "Launch Create dependency image for $VERSION_NUMBER"
+
+git clone --branch master --depth 1 https://github.com/cytomine/pims ./ci/app
+
+mkdir -p ./ci/app/plugins/pims-plugin-format-bioformats/
+cp -r ./pims_plugin_format_bioformats ./ci/app/plugins/pims-plugin-format-bioformats/
+#cp -r ./pims_plugin_format_bioformats.egg-info ./ci/app/plugins/pims-plugin-format-bioformats/
+cp -r ./tests ./ci/app/plugins/pims-plugin-format-bioformats/
+cp ./setup.py ./ci/app/plugins/pims-plugin-format-bioformats/
+
+#git clone https://github.com/cytomine/pims-plugin-format-openslide ./ci/app/plugins/pims-plugin-format-openslide
+
+docker build --rm -f scripts/docker/Dockerfile-dependencies -t  cytomine/pims-plugin-format-bioformats-dependencies:v$VERSION_NUMBER .

scripts/ciTest.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -o xtrace
+set -o errexit
+
+echo "************************************** Launch tests ******************************************"
+
+file='./ci/version'
+VERSION_NUMBER=$(<"$file")
+
+echo "Launch tests for $VERSION_NUMBER"
+mkdir "$PWD"/ci/test-reports
+touch "$PWD"/ci/test-reports/pytest_unit.xml
+docker build --rm -f scripts/docker/Dockerfile-test --build-arg VERSION_NUMBER=$VERSION_NUMBER -t  cytomine/pims-plugin-format-bioformats-test .
+
+containerIdBioformat=$(docker create --name bioformat -v /data/images:/data/images -v /data/pims:/data/pims -e BIOFORMAT_PORT=4321 --restart=unless-stopped cytomine/bioformat:v3.1.0 )
+docker start $containerIdBioformat
+
+containerId=$(docker create --link bioformat:bioformat -v /data/pims:/data/pims -v "$PWD"/ci/test-reports:/app/ci/test-reports -v /tmp/uploaded:/tmp/uploaded -v /tmp/test-pims:/tmp/test-pims cytomine/pims-plugin-format-bioformats-test )
+
+docker start -ai  $containerId
+docker rm $containerId
+docker stop $containerIdBioformat
+docker rm $containerIdBioformat

scripts/docker/Dockerfile-dependencies

@@ -0,0 +1,146 @@
+FROM ubuntu:20.04
+
+ENV LANG C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+
+ARG PY_VERSION=3.8
+
+RUN apt-get -y update && apt-get -y install --no-install-recommends --no-install-suggests \
+        `# Essentials` \
+        automake \
+        build-essential \
+        ca-certificates \
+        cmake \
+        git \
+        gcc \
+        net-tools \
+        python${PY_VERSION} \
+        python${PY_VERSION}-dev \
+        python${PY_VERSION}-distutils \
+        wget \
+        software-properties-common \
+        `# Vips dependencies` \
+        pkg-config \
+        glib2.0-dev \
+        libexpat1-dev \
+        libtiff5-dev \
+        libjpeg-turbo8 \
+        libgsf-1-dev \
+        libexif-dev \
+        libvips-dev \
+        orc-0.4-dev \
+        libwebp-dev \
+        liblcms2-dev \
+        libpng-dev \
+        gobject-introspection \
+        `# Other tools` \
+        libimage-exiftool-perl
+
+RUN cd /usr/bin && \
+    ln -s python${PY_VERSION} python
+
+# Official pip install: https://pip.pypa.io/en/stable/installation/#get-pip-py
+RUN cd /tmp && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py && \
+    rm -rf get-pip.py
+
+# openjpeg 2.4 is required by vips (J2000 support)
+ARG OPENJPEG_VERSION=2.4.0
+ARG OPENJPEG_URL=https://github.com/uclouvain/openjpeg/archive
+RUN cd /usr/local/src && \
+    wget ${OPENJPEG_URL}/v${OPENJPEG_VERSION}/openjpeg-${OPENJPEG_VERSION}.tar.gz && \
+    tar -zxvf openjpeg-${OPENJPEG_VERSION}.tar.gz && \
+    rm -rf openjpeg-${OPENJPEG_VERSION}.tar.gz && \
+    cd openjpeg-${OPENJPEG_VERSION} && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_STATIC_LIBS=ON .. && \
+    make && \
+    make install && \
+    make clean && \
+    ldconfig
+
+# Download plugins
+WORKDIR /app
+COPY ./ci/app/docker/plugins.py /app/plugins.py
+
+ARG PLUGIN_CSV
+# ="enabled,name,git_url,git_branch\n"
+ENV PLUGIN_INSTALL_PATH /app/plugins
+RUN python3 plugins.py \
+   --plugin_csv ${PLUGIN_CSV} \
+   --install_path ${PLUGIN_INSTALL_PATH} \
+   --method download
+
+# Run before_vips() from plugins prerequisites
+RUN python3 plugins.py \
+   --plugin_csv ${PLUGIN_CSV} \
+   --install_path ${PLUGIN_INSTALL_PATH} \
+   --method dependencies_before_vips
+
+# vips
+ARG VIPS_VERSION=8.11.2
+ARG VIPS_URL=https://github.com/libvips/libvips/releases/download
+RUN cd /usr/local/src && \
+    wget ${VIPS_URL}/v${VIPS_VERSION}/vips-${VIPS_VERSION}.tar.gz && \
+    tar -zxvf vips-${VIPS_VERSION}.tar.gz && \
+    rm -rf vips-${VIPS_VERSION}.tar.gz && \
+    cd vips-${VIPS_VERSION} && \
+    ./configure && \
+    make V=0 && \
+    make install && \
+    ldconfig
+
+# Run before_python() from plugins prerequisites
+RUN python3 plugins.py \
+   --plugin_csv ${PLUGIN_CSV} \
+   --install_path ${PLUGIN_INSTALL_PATH} \
+   --method dependencies_before_python
+
+
+# Cleaning. Cannot be done before as plugin prerequisites could use apt-get.
+RUN rm -rf /var/lib/apt/lists/*
+
+# Install python requirements
+COPY ./ci/app/requirements.txt /app/requirements.txt
+
+ARG GUNICORN_VERSION=20.1.0
+RUN pip3 install gunicorn==${GUNICORN_VERSION} && \
+    pip3 install -r requirements.txt && \
+    python3 plugins.py \
+   --plugin_csv ${PLUGIN_CSV} \
+   --install_path ${PLUGIN_INSTALL_PATH} \
+   --method install
+
+#Install plugins
+# COPY ./ci/app/plugins/pims-plugin-format-openslide /app/plugins/pims-plugin-format-openslide/
+COPY ./ci/app/plugins/pims-plugin-format-bioformats /app/plugins/pims-plugin-format-bioformats/
+# RUN pip3 install /app/plugins/pims-plugin-format-openslide
+RUN pip3 install /app/plugins/pims-plugin-format-bioformats
+
+# Prestart configuration
+RUN touch /tmp/addHosts.sh
+COPY ./ci/app/docker/prestart.sh /app/prestart.sh
+RUN chmod +x /app/prestart.sh
+
+# Add default config
+COPY ./ci/app/pims-config.env /app/pims-config.env
+COPY ./ci/app/logging-prod.yml /app/logging-prod.yml
+COPY ./ci/app/docker/gunicorn_conf.py /app/gunicorn_conf.py
+
+COPY ./ci/app/docker/start.sh /start.sh
+RUN chmod +x /start.sh
+
+COPY ./ci/app/docker/start-reload.sh /start-reload.sh
+RUN chmod +x /start-reload.sh
+
+ENV PYTHONPATH=/app
+
+# Add app
+COPY ./ci/app/pims /app/pims
+COPY ./ci/app/setup.py /app
+ENV MODULE_NAME="pims.application"
+
+ENV PORT=5000
+EXPOSE ${PORT}
+CMD ["/start.sh"]

scripts/docker/Dockerfile-final

@@ -0,0 +1,7 @@
+ARG  VERSION_NUMBER
+FROM cytomine/pims-plugin-format-bioformats-dependencies:v$VERSION_NUMBER
+
+ENV MODULE_NAME="pims.application"
+ENV PORT=5000
+EXPOSE ${PORT}
+CMD ["/start.sh"]

scripts/docker/Dockerfile-test

@@ -0,0 +1,14 @@
+ARG  VERSION_NUMBER
+FROM cytomine/pims-plugin-format-bioformats-dependencies:v$VERSION_NUMBER
+
+RUN pip3 install pytest
+RUN pip3 install -e /app
+
+
+COPY ./ci/app/tests /app/tests
+
+RUN ls /app/
+
+RUN ls /app/tests
+
+CMD pytest -rP /app/plugins/pims-plugin-format-bioformats/tests --junit-xml=ci/test-reports/pytest_unit.xml