Merge main branch into dev
Showing 24 of 55 files from the diff.
pathml/ml/__init__.py
changed.
pathml/core/tiles.py
changed.
pathml/ml/dataset.py
changed.
pathml/core/__init__.py
changed.
pathml/core/slide_backends.py
changed.
pathml/core/slide_data.py
changed.
pathml/preprocessing/tiling.py
changed.
pathml/preprocessing/transforms.py
changed.
pathml/preprocessing/__init__.py
changed.
pathml/datasets/__init__.py
changed.
pathml/core/masks.py
changed.
pathml/core/slide_dataset.py
changed.
pathml/ml/utils.py
changed.
pathml/_logging.py
changed.
pathml/core/h5managers.py
changed.
pathml/ml/hovernet.py
changed.
pathml/_version.py
changed.
pathml/datasets/deepfocus.py
changed.
pathml/core/utils.py
changed.
pathml/preprocessing/pipeline.py
changed.
pathml/core/slide_types.py
changed.
pathml/datasets/pannuke.py
changed.
pathml/datasets/utils.py
changed.
pathml/core/tile.py
changed.
Other files ignored by Codecov
tests/datasets_tests/test_dataset_utils.py
has changed.
CITATION.cff
is new.
tests/core_tests/test_slide_data.py
has changed.
pathml/__init__.py
has changed.
.flake8
is new.
pathml/utils.py
has changed.
tests/conftest.py
has changed.
tests/benchmark/he_benchmark.py
has changed.
tests/test_utils.py
has changed.
tests/core_tests/test_slide_backends.py
has changed.
tests/preprocessing_tests/test_transforms.py
has changed.
tests/test_version.py
has changed.
pyproject.toml
has changed.
tests/test_manuscript_urls.py
has changed.
tests/preprocessing_tests/test_pipeline.py
has changed.
tests/ml_tests/test_hovernet.py
has changed.
.pre-commit-config.yaml
has changed.
tests/ml_tests/test_dataset.py
has changed.
tests/core_tests/conftest.py
has changed.
tests/datasets_tests/test_pannuke.py
has changed.
tests/core_tests/test_h5managers.py
has changed.
tests/core_tests/test_masks.py
has changed.
tests/core_tests/test_slide_dataset.py
has changed.
tests/datasets_tests/test_deepblur.py
has changed.
README.md
has changed.
tests/test_logging.py
has changed.
tests/preprocessing_tests/test_tiling.py
has changed.
tests/integration_tests/test_pipeline_running.py
has changed.
tests/core_tests/test_tile.py
has changed.
tests/core_tests/test_tiles.py
has changed.
@@ -3,13 +3,9 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | - | import os |
|
7 | 6 | import reprlib |
|
8 | 7 | from collections import OrderedDict |
|
9 | - | from pathlib import Path |
|
10 | - | from loguru import logger |
|
11 | 8 | ||
12 | - | import h5py |
|
13 | 9 | import pathml.core.h5managers |
|
14 | 10 | import pathml.core.masks |
|
15 | 11 | import pathml.core.tile |
@@ -39,7 +35,7 @@
Loading
39 | 35 | f"Tiles expects a list of type Tile but was given {type(tile)}" |
|
40 | 36 | ) |
|
41 | 37 | if tile.coords is None: |
|
42 | - | raise ValueError(f"tiles must contain valid coords") |
|
38 | + | raise ValueError("tiles must contain valid coords") |
|
43 | 39 | coords = tile.coords |
|
44 | 40 | tiledictionary[coords] = tile |
|
45 | 41 | self._tiles = OrderedDict(tiledictionary) |
@@ -3,7 +3,6 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | - | from loguru import logger |
|
7 | 6 | import h5py |
|
8 | 7 | import numpy as np |
|
9 | 8 | import torch |
@@ -52,7 +51,7 @@
Loading
52 | 51 | self.h5 = h5py.File(self.file_path, "r") |
|
53 | 52 | ||
54 | 53 | k = self.tile_keys[ix] |
|
55 | - | ### this part copied from h5manager.get_tile() |
|
54 | + | # this part copied from h5manager.get_tile() |
|
56 | 55 | tile_image = self.h5["tiles"][str(k)]["array"][:] |
|
57 | 56 | ||
58 | 57 | # get corresponding masks if there are masks |
@@ -4,16 +4,16 @@
Loading
4 | 4 | """ |
|
5 | 5 | ||
6 | 6 | from .masks import Masks |
|
7 | - | from .slide_backends import OpenSlideBackend, BioFormatsBackend, DICOMBackend |
|
7 | + | from .slide_backends import BioFormatsBackend, DICOMBackend, OpenSlideBackend |
|
8 | 8 | from .slide_data import ( |
|
9 | - | SlideData, |
|
9 | + | CODEXSlide, |
|
10 | 10 | HESlide, |
|
11 | + | IHCSlide, |
|
11 | 12 | MultiparametricSlide, |
|
13 | + | SlideData, |
|
12 | 14 | VectraSlide, |
|
13 | - | CODEXSlide, |
|
14 | - | IHCSlide, |
|
15 | 15 | ) |
|
16 | 16 | from .slide_dataset import SlideDataset |
|
17 | + | from .slide_types import SlideType, types |
|
17 | 18 | from .tile import Tile |
|
18 | 19 | from .tiles import Tiles |
|
19 | - | from .slide_types import SlideType, types |
@@ -4,15 +4,11 @@
Loading
4 | 4 | """ |
|
5 | 5 | ||
6 | 6 | from io import BytesIO |
|
7 | - | from typing import Tuple |
|
8 | 7 | ||
9 | 8 | import numpy as np |
|
10 | 9 | import openslide |
|
11 | - | from loguru import logger |
|
12 | - | import pathml.core |
|
13 | - | import pathml.core.tile |
|
14 | 10 | from javabridge.jutil import JavaException |
|
15 | - | from pathml.utils import pil_to_rgb |
|
11 | + | from loguru import logger |
|
16 | 12 | from PIL import Image |
|
17 | 13 | from pydicom.dataset import Dataset |
|
18 | 14 | from pydicom.encaps import get_frame_offsets |
@@ -22,6 +18,10 @@
Loading
22 | 18 | from pydicom.uid import UID |
|
23 | 19 | from scipy.ndimage import zoom |
|
24 | 20 | ||
21 | + | import pathml.core |
|
22 | + | import pathml.core.tile |
|
23 | + | from pathml.utils import pil_to_rgb |
|
24 | + | ||
25 | 25 | try: |
|
26 | 26 | import bioformats |
|
27 | 27 | import javabridge |
@@ -29,7 +29,7 @@
Loading
29 | 29 | except ImportError: |
|
30 | 30 | logger.exception("Unable to import bioformats, javabridge") |
|
31 | 31 | raise Exception( |
|
32 | - | f"Installation of PathML not complete. Please install openjdk8, bioformats, and javabridge:\nconda install openjdk==8.0.152\npip install javabridge==1.0.19 python-bioformats==4.0.0\nFor detailed installation instructions, please see https://github.com/Dana-Farber-AIOS/pathml/" |
|
32 | + | "Installation of PathML not complete. Please install openjdk8, bioformats, and javabridge:\nconda install openjdk==8.0.152\npip install javabridge==1.0.19 python-bioformats==4.0.0\nFor detailed installation instructions, please see https://github.com/Dana-Farber-AIOS/pathml/" |
|
33 | 33 | ) |
|
34 | 34 | ||
35 | 35 |
@@ -330,7 +330,8 @@
Loading
330 | 330 | try: |
|
331 | 331 | self.pixel_dtype = pixel_dtype_map[ome_pixeltype] |
|
332 | 332 | logger.info(f"Found corresponding dtype: {self.pixel_dtype}") |
|
333 | - | except: |
|
333 | + | # TODO: change to specific exception |
|
334 | + | except Exception: |
|
334 | 335 | logger.exception("datatype from metadata not found in pixel_dtype_map") |
|
335 | 336 | raise Exception( |
|
336 | 337 | f"pixel type '{ome_pixeltype}' detected from OME metadata not recognized." |
@@ -411,7 +412,7 @@
Loading
411 | 412 | f"input size {size} invalid. Must be a tuple of integer coordinates of len<2" |
|
412 | 413 | ) |
|
413 | 414 | if series_as_channels: |
|
414 | - | logger.info(f"using series_as_channels=True") |
|
415 | + | logger.info("using series_as_channels=True") |
|
415 | 416 | if level != 0: |
|
416 | 417 | logger.exception( |
|
417 | 418 | f"When series_as_channels=True, must use level=0. Input 'level={level}' invalid." |
@@ -490,7 +491,7 @@
Loading
490 | 491 | f"Scaling image to [0, 1] by dividing by {(2 ** (8 * self.pixel_dtype.itemsize))}" |
|
491 | 492 | ) |
|
492 | 493 | # then scale to [0-255] and convert to 8 bit |
|
493 | - | array_scaled = array_scaled * 2 ** 8 |
|
494 | + | array_scaled = array_scaled * 2**8 |
|
494 | 495 | return array_scaled.astype(np.uint8) |
|
495 | 496 | ||
496 | 497 | def get_thumbnail(self, size=None): |
@@ -507,7 +508,7 @@
Loading
507 | 508 | shape = data.slide.get_image_shape() |
|
508 | 509 | thumb = data.slide.get_thumbnail(size=(1000,1000, shape[2], shape[3], shape[4])) |
|
509 | 510 | """ |
|
510 | - | assert isinstance(size, (tuple, type(None))), f"Size must be a tuple of ints." |
|
511 | + | assert isinstance(size, (tuple, type(None))), "Size must be a tuple of ints." |
|
511 | 512 | if size is not None: |
|
512 | 513 | if len(size) != len(self.shape): |
|
513 | 514 | size = size + self.shape[len(size) :] |
@@ -520,7 +521,7 @@
Loading
520 | 521 | ratio = tuple([x / y for x, y in zip(size, self.shape)]) |
|
521 | 522 | assert ( |
|
522 | 523 | ratio[3] == 1 |
|
523 | - | ), f"cannot interpolate between fluor channels, resampling doesn't apply, fix size[3]" |
|
524 | + | ), "cannot interpolate between fluor channels, resampling doesn't apply, fix size[3]" |
|
524 | 525 | image_array = zoom(array, ratio) |
|
525 | 526 | return image_array |
|
526 | 527 |
@@ -768,7 +769,7 @@
Loading
768 | 769 | Returns: |
|
769 | 770 | np.ndarray: image at the specified region |
|
770 | 771 | """ |
|
771 | - | assert level == 0 or level is None, f"dicom does not support levels" |
|
772 | + | assert level == 0 or level is None, "dicom does not support levels" |
|
772 | 773 | # check inputs first |
|
773 | 774 | # check location |
|
774 | 775 | if isinstance(location, tuple): |
@@ -887,7 +888,7 @@
Loading
887 | 888 | Yields: |
|
888 | 889 | pathml.core.tile.Tile: Extracted Tile object |
|
889 | 890 | """ |
|
890 | - | assert level == 0 or level is None, f"dicom does not support levels" |
|
891 | + | assert level == 0 or level is None, "dicom does not support levels" |
|
891 | 892 | for i in range(self.n_frames): |
|
892 | 893 | ||
893 | 894 | if not pad: |
@@ -8,11 +8,12 @@
Loading
8 | 8 | from pathlib import Path |
|
9 | 9 | ||
10 | 10 | import anndata |
|
11 | - | from loguru import logger |
|
12 | 11 | import dask.distributed |
|
13 | 12 | import h5py |
|
14 | 13 | import matplotlib.pyplot as plt |
|
15 | 14 | import numpy as np |
|
15 | + | from loguru import logger |
|
16 | + | ||
16 | 17 | import pathml.core |
|
17 | 18 | import pathml.preprocessing.pipeline |
|
18 | 19 | from pathml.core.slide_types import SlideType |
@@ -213,7 +214,8 @@
Loading
213 | 214 | out.append(f"image shape: {self.shape}") |
|
214 | 215 | try: |
|
215 | 216 | nlevels = self.slide.level_count |
|
216 | - | except: |
|
217 | + | # TODO: change to specific exception |
|
218 | + | except Exception: |
|
217 | 219 | nlevels = 1 |
|
218 | 220 | out.append(f"number of levels: {nlevels}") |
|
219 | 221 | out.append(repr(self.tiles)) |
@@ -229,7 +231,7 @@
Loading
229 | 231 | if self.counts: |
|
230 | 232 | out.append(f"counts matrix of shape {self.counts.shape}") |
|
231 | 233 | else: |
|
232 | - | out.append(f"counts=None") |
|
234 | + | out.append("counts=None") |
|
233 | 235 | ||
234 | 236 | out = ",\n\t".join(out) |
|
235 | 237 | out += ")" |
@@ -279,7 +281,7 @@
Loading
279 | 281 | # in this case, tiles already exist |
|
280 | 282 | if not overwrite_existing_tiles: |
|
281 | 283 | raise Exception( |
|
282 | - | f"Slide already has tiles. Running the pipeline will overwrite the existing tiles. Use overwrite_existing_tiles=True to force overwriting existing tiles." |
|
284 | + | "Slide already has tiles. Running the pipeline will overwrite the existing tiles. Use overwrite_existing_tiles=True to force overwriting existing tiles." |
|
283 | 285 | ) |
|
284 | 286 | else: |
|
285 | 287 | # delete all existing tiles |
@@ -448,10 +450,11 @@
Loading
448 | 450 | """ |
|
449 | 451 | try: |
|
450 | 452 | thumbnail = self.slide.get_thumbnail(size=(500, 500)) |
|
451 | - | except: |
|
453 | + | # TODO: change to specific exception |
|
454 | + | except Exception: |
|
452 | 455 | if not self.slide: |
|
453 | 456 | raise NotImplementedError( |
|
454 | - | f"Plotting only supported via backend, but SlideData has no backend." |
|
457 | + | "Plotting only supported via backend, but SlideData has no backend." |
|
455 | 458 | ) |
|
456 | 459 | else: |
|
457 | 460 | raise NotImplementedError( |
@@ -477,7 +480,7 @@
Loading
477 | 480 | self.tiles.h5manager.counts = value |
|
478 | 481 | else: |
|
479 | 482 | raise AttributeError( |
|
480 | - | f"cannot assign counts slidedata contains no tiles, first generate tiles" |
|
483 | + | "cannot assign counts slidedata contains no tiles, first generate tiles" |
|
481 | 484 | ) |
|
482 | 485 | ||
483 | 486 | def write(self, path): |
@@ -4,13 +4,16 @@
Loading
4 | 4 | """ |
|
5 | 5 | ||
6 | 6 | import os |
|
7 | - | from warnings import warn |
|
8 | 7 | ||
9 | - | from loguru import logger |
|
10 | 8 | import anndata |
|
11 | 9 | import cv2 |
|
12 | 10 | import numpy as np |
|
13 | 11 | import pandas as pd |
|
12 | + | from loguru import logger |
|
13 | + | from skimage import restoration |
|
14 | + | from skimage.exposure import equalize_adapthist, equalize_hist, rescale_intensity |
|
15 | + | from skimage.measure import regionprops_table |
|
16 | + | ||
14 | 17 | import pathml.core |
|
15 | 18 | import pathml.core.slide_data |
|
16 | 19 | from pathml.utils import ( |
@@ -20,9 +23,7 @@
Loading
20 | 23 | RGB_to_OD, |
|
21 | 24 | normalize_matrix_cols, |
|
22 | 25 | ) |
|
23 | - | from skimage import restoration |
|
24 | - | from skimage.exposure import equalize_adapthist, equalize_hist, rescale_intensity |
|
25 | - | from skimage.measure import regionprops_table |
|
26 | + | ||
26 | 27 | ||
27 | 28 | # Base class |
|
28 | 29 | class Transform: |
@@ -647,10 +648,10 @@
Loading
647 | 648 | ||
648 | 649 | if stain_estimation_method.lower() == "vahadane": |
|
649 | 650 | try: |
|
650 | - | import spams |
|
651 | + | import spams # noqa: F401 |
|
651 | 652 | except (ImportError, ModuleNotFoundError): |
|
652 | 653 | raise Exception( |
|
653 | - | f"Vahadane method requires `spams` package to be installed" |
|
654 | + | "Vahadane method requires `spams` package to be installed" |
|
654 | 655 | ) |
|
655 | 656 | ||
656 | 657 | self.target = target.lower() |
@@ -743,7 +744,7 @@
Loading
743 | 744 | try: |
|
744 | 745 | import spams |
|
745 | 746 | except (ImportError, ModuleNotFoundError): |
|
746 | - | raise Exception(f"Vahadane method requires `spams` package to be installed") |
|
747 | + | raise Exception("Vahadane method requires `spams` package to be installed") |
|
747 | 748 | # convert to Optical Density (OD) space |
|
748 | 749 | image_OD = RGB_to_OD(image) |
|
749 | 750 | # reshape to (M*N)x3 |
@@ -847,7 +848,7 @@
Loading
847 | 848 | try: |
|
848 | 849 | import spams |
|
849 | 850 | except (ImportError, ModuleNotFoundError): |
|
850 | - | raise Exception(f"Vahadane method requires `spams` package to be installed") |
|
851 | + | raise Exception("Vahadane method requires `spams` package to be installed") |
|
851 | 852 | image_OD = RGB_to_OD(image).reshape(-1, 3) |
|
852 | 853 | ||
853 | 854 | # Get concentrations of each stain at each pixel |
@@ -1230,7 +1231,7 @@
Loading
1230 | 1231 | if psfparameters: |
|
1231 | 1232 | assert ( |
|
1232 | 1233 | psf is None |
|
1233 | - | ), f"you passed an empirical psf, cannot simultaneously use theoretical psf" |
|
1234 | + | ), "you passed an empirical psf, cannot simultaneously use theoretical psf" |
|
1234 | 1235 | self.psfparameters = psfparameters |
|
1235 | 1236 | self.iterations = iterations |
|
1236 | 1237 |
@@ -1318,10 +1319,10 @@
Loading
1318 | 1319 | ): |
|
1319 | 1320 | assert isinstance( |
|
1320 | 1321 | nuclear_channel, int |
|
1321 | - | ), f"nuclear_channel must be an int indicating index" |
|
1322 | + | ), "nuclear_channel must be an int indicating index" |
|
1322 | 1323 | assert isinstance( |
|
1323 | 1324 | cytoplasm_channel, int |
|
1324 | - | ), f"cytoplasm_channel must be an int indicating index" |
|
1325 | + | ), "cytoplasm_channel must be an int indicating index" |
|
1325 | 1326 | self.nuclear_channel = nuclear_channel |
|
1326 | 1327 | self.cytoplasm_channel = cytoplasm_channel |
|
1327 | 1328 | self.image_resolution = image_resolution |
@@ -1335,21 +1336,21 @@
Loading
1335 | 1336 | ||
1336 | 1337 | if model.lower() == "mesmer": |
|
1337 | 1338 | try: |
|
1338 | - | from deepcell.applications import Mesmer |
|
1339 | + | from deepcell.applications import Mesmer # noqa: F401 |
|
1339 | 1340 | except ImportError: |
|
1340 | 1341 | logger.warning( |
|
1341 | 1342 | "The Mesmer model in SegmentMIF requires extra libraries to be installed.\nYou can install these via pip using:\npip install deepcell" |
|
1342 | 1343 | ) |
|
1343 | 1344 | raise ImportError( |
|
1344 | - | f"The Mesmer model in SegmentMIF requires deepcell to be installed" |
|
1345 | + | "The Mesmer model in SegmentMIF requires deepcell to be installed" |
|
1345 | 1346 | ) from None |
|
1346 | 1347 | self.model = model.lower() |
|
1347 | 1348 | elif model.lower() == "cellpose": |
|
1348 | 1349 | """from cellpose import models |
|
1349 | 1350 | self.model = models.Cellpose(gpu=self.gpu, model_type='cyto')""" |
|
1350 | - | raise NotImplementedError(f"Cellpose model not currently supported") |
|
1351 | + | raise NotImplementedError("Cellpose model not currently supported") |
|
1351 | 1352 | else: |
|
1352 | - | raise ValueError(f"currently only supports mesmer model") |
|
1353 | + | raise ValueError("currently only supports mesmer model") |
|
1353 | 1354 | ||
1354 | 1355 | def __repr__(self): |
|
1355 | 1356 | return ( |
@@ -1491,7 +1492,8 @@
Loading
1491 | 1492 | counts.layers["max_intensity"] = max_intensities |
|
1492 | 1493 | try: |
|
1493 | 1494 | counts.obsm["spatial"] = np.array(counts.obs[["x", "y"]]) |
|
1494 | - | except: |
|
1495 | + | # TODO: change to specific exception |
|
1496 | + | except Exception: |
|
1495 | 1497 | logger.warning("did not log coordinates in obsm") |
|
1496 | 1498 | return counts |
|
1497 | 1499 |
@@ -1522,7 +1524,7 @@
Loading
1522 | 1524 | pass |
|
1523 | 1525 | ||
1524 | 1526 | def __repr__(self): |
|
1525 | - | return f"CollapseRunsVectra()" |
|
1527 | + | return "CollapseRunsVectra()" |
|
1526 | 1528 | ||
1527 | 1529 | def F(self, image): |
|
1528 | 1530 | image = np.squeeze(image) |
@@ -5,24 +5,24 @@
Loading
5 | 5 | ||
6 | 6 | from .pipeline import Pipeline |
|
7 | 7 | from .transforms import ( |
|
8 | + | AdaptiveHistogramEqualization, |
|
8 | 9 | BinaryThreshold, |
|
9 | 10 | BoxBlur, |
|
11 | + | CollapseRunsCODEX, |
|
12 | + | CollapseRunsVectra, |
|
10 | 13 | ForegroundDetection, |
|
11 | 14 | GaussianBlur, |
|
15 | + | HistogramEqualization, |
|
16 | + | LabelArtifactTileHE, |
|
17 | + | LabelWhiteSpaceHE, |
|
12 | 18 | MedianBlur, |
|
13 | - | MorphOpen, |
|
14 | 19 | MorphClose, |
|
20 | + | MorphOpen, |
|
15 | 21 | NucleusDetectionHE, |
|
22 | + | QuantifyMIF, |
|
23 | + | RescaleIntensity, |
|
24 | + | SegmentMIF, |
|
16 | 25 | StainNormalizationHE, |
|
17 | 26 | SuperpixelInterpolation, |
|
18 | 27 | TissueDetectionHE, |
|
19 | - | LabelArtifactTileHE, |
|
20 | - | LabelWhiteSpaceHE, |
|
21 | - | SegmentMIF, |
|
22 | - | QuantifyMIF, |
|
23 | - | CollapseRunsVectra, |
|
24 | - | CollapseRunsCODEX, |
|
25 | - | RescaleIntensity, |
|
26 | - | HistogramEqualization, |
|
27 | - | AdaptiveHistogramEqualization, |
|
28 | 28 | ) |
@@ -3,13 +3,10 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | - | import numpy as np |
|
7 | - | import os |
|
8 | - | from pathlib import Path |
|
9 | - | from collections import OrderedDict |
|
10 | - | import h5py |
|
11 | 6 | import reprlib |
|
12 | - | from loguru import logger |
|
7 | + | from collections import OrderedDict |
|
8 | + | ||
9 | + | import numpy as np |
|
13 | 10 | ||
14 | 11 | import pathml.core.h5managers |
|
15 | 12 |
@@ -5,10 +5,9 @@
Loading
5 | 5 | ||
6 | 6 | import reprlib |
|
7 | 7 | from pathlib import Path |
|
8 | - | from loguru import logger |
|
9 | 8 | ||
10 | 9 | import dask.distributed |
|
11 | - | from torch.utils.data import ConcatDataset |
|
10 | + | from loguru import logger |
|
12 | 11 | ||
13 | 12 | ||
14 | 13 | class SlideDataset: |
@@ -88,6 +87,6 @@
Loading
88 | 87 | slide_path = d / (slide.name + ".h5path") |
|
89 | 88 | else: |
|
90 | 89 | raise ValueError( |
|
91 | - | f"slide does not have a .name attribute. Must supply a 'filenames' argument." |
|
90 | + | "slide does not have a .name attribute. Must supply a 'filenames' argument." |
|
92 | 91 | ) |
|
93 | 92 | slide.write(slide_path) |
@@ -3,11 +3,11 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | + | import numpy as np |
|
7 | + | ||
6 | 8 | # Utilities for ML module |
|
7 | 9 | import torch |
|
8 | 10 | from torch.nn import functional as F |
|
9 | - | import numpy as np |
|
10 | - | from loguru import logger |
|
11 | 11 | ||
12 | 12 | ||
13 | 13 | def center_crop_im_batch(batch, dims, batch_order="BCHW"): |
@@ -43,7 +43,7 @@
Loading
43 | 43 | elif batch_order == "BCHW": |
|
44 | 44 | batch_cropped = batch[:, :, crop_t:-crop_b, crop_l:-crop_r] |
|
45 | 45 | else: |
|
46 | - | raise Exception(f"Input batch order not valid") |
|
46 | + | raise Exception("Input batch order not valid") |
|
47 | 47 | ||
48 | 48 | return batch_cropped |
|
49 | 49 |
@@ -160,7 +160,7 @@
Loading
160 | 160 | # make sure that everything is correct so that transform is correctly applied |
|
161 | 161 | assert all( |
|
162 | 162 | [v == "mask" for v in targets.values()] |
|
163 | - | ), f"error all values in transform.additional_targets must be 'mask'." |
|
163 | + | ), "error all values in transform.additional_targets must be 'mask'." |
|
164 | 164 | ||
165 | 165 | def transform_out(*args, **kwargs): |
|
166 | 166 | mask = kwargs.pop("mask") |
@@ -3,15 +3,14 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | - | import itertools |
|
7 | 6 | import os |
|
8 | 7 | import tempfile |
|
9 | - | from collections import OrderedDict |
|
10 | 8 | ||
11 | 9 | import anndata |
|
12 | - | from loguru import logger |
|
13 | 10 | import h5py |
|
14 | 11 | import numpy as np |
|
12 | + | from loguru import logger |
|
13 | + | ||
15 | 14 | import pathml.core |
|
16 | 15 | import pathml.core.masks |
|
17 | 16 | import pathml.core.tile |
@@ -35,10 +34,10 @@
Loading
35 | 34 | if h5path: |
|
36 | 35 | assert ( |
|
37 | 36 | not slidedata |
|
38 | - | ), f"if creating h5pathmanager from h5path, slidedata should not be required" |
|
37 | + | ), "if creating h5pathmanager from h5path, slidedata should not be required" |
|
39 | 38 | assert check_valid_h5path_format( |
|
40 | 39 | h5path |
|
41 | - | ), f"h5path must conform to .h5path standard, see documentation" |
|
40 | + | ), "h5path must conform to .h5path standard, see documentation" |
|
42 | 41 | # copy h5path into self.h5 |
|
43 | 42 | for ds in h5path.keys(): |
|
44 | 43 | if ds in ["fields", "masks", "tiles"]: |
@@ -52,19 +51,19 @@
Loading
52 | 51 | ) |
|
53 | 52 | ||
54 | 53 | else: |
|
55 | - | assert slidedata, f"must pass slidedata object to create h5path" |
|
54 | + | assert slidedata, "must pass slidedata object to create h5path" |
|
56 | 55 | # initialize h5path file hierarchy |
|
57 | 56 | # fields |
|
58 | 57 | fieldsgroup = self.h5.create_group("fields") |
|
59 | 58 | # name, shape, labels |
|
60 | 59 | fieldsgroup.attrs["name"] = slidedata.name |
|
61 | 60 | fieldsgroup.attrs["shape"] = slidedata.slide.get_image_shape() |
|
62 | - | labelsgroup = self.h5["fields"].create_group("labels") |
|
61 | + | self.h5["fields"].create_group("labels") |
|
63 | 62 | if slidedata.labels: |
|
64 | 63 | for key, label in slidedata.labels.items(): |
|
65 | 64 | self.h5["fields/labels"].attrs[key] = label |
|
66 | 65 | # slidetype |
|
67 | - | slidetypegroup = self.h5["fields"].create_group("slide_type") |
|
66 | + | self.h5["fields"].create_group("slide_type") |
|
68 | 67 | if slidedata.slide_type: |
|
69 | 68 | for key, val in slidedata.slide_type.asdict().items(): |
|
70 | 69 | self.h5["fields/slide_type"].attrs[key] = val |
@@ -75,9 +74,9 @@
Loading
75 | 74 | # initialize stride with 0 |
|
76 | 75 | tilesgroup.attrs["tile_stride"] = b"(0, 0)" |
|
77 | 76 | # masks |
|
78 | - | masksgroup = self.h5.create_group("masks") |
|
77 | + | self.h5.create_group("masks") |
|
79 | 78 | # counts |
|
80 | - | countsgroup = self.h5.create_group("counts") |
|
79 | + | self.h5.create_group("counts") |
|
81 | 80 | ||
82 | 81 | slide_type_dict = { |
|
83 | 82 | key: val for key, val in self.h5["fields/slide_type"].attrs.items() |
@@ -150,7 +149,7 @@
Loading
150 | 149 | if tile.masks: |
|
151 | 150 | # create a group to hold tile-level masks |
|
152 | 151 | if "masks" not in self.h5["tiles"][str(tile.coords)].keys(): |
|
153 | - | masksgroup = self.h5["tiles"][str(tile.coords)].create_group("masks") |
|
152 | + | self.h5["tiles"][str(tile.coords)].create_group("masks") |
|
154 | 153 | ||
155 | 154 | # add tile-level masks |
|
156 | 155 | for key, mask in tile.masks.items(): |
@@ -168,7 +167,7 @@
Loading
168 | 167 | self.h5["tiles"][str(tile.coords)].attrs["name"] = ( |
|
169 | 168 | str(tile.name) if tile.name else 0 |
|
170 | 169 | ) |
|
171 | - | tilelabelsgroup = self.h5["tiles"][str(tile.coords)].create_group("labels") |
|
170 | + | self.h5["tiles"][str(tile.coords)].create_group("labels") |
|
172 | 171 | if tile.labels: |
|
173 | 172 | for key, val in tile.labels.items(): |
|
174 | 173 | self.h5["tiles"][str(tile.coords)]["labels"].attrs[key] = val |
@@ -198,7 +197,7 @@
Loading
198 | 197 | Tile(pathml.core.tile.Tile) |
|
199 | 198 | """ |
|
200 | 199 | if isinstance(item, bool): |
|
201 | - | raise KeyError(f"invalid key, pass str or tuple") |
|
200 | + | raise KeyError("invalid key, pass str or tuple") |
|
202 | 201 | if isinstance(item, (str, tuple)): |
|
203 | 202 | item = str(item) |
|
204 | 203 | if item not in self.h5["tiles"].keys(): |
@@ -246,7 +245,7 @@
Loading
246 | 245 | Remove tile from self.h5 by key. |
|
247 | 246 | """ |
|
248 | 247 | if not isinstance(key, (str, tuple)): |
|
249 | - | raise KeyError(f"key must be str or tuple, check valid keys in repr") |
|
248 | + | raise KeyError("key must be str or tuple, check valid keys in repr") |
|
250 | 249 | if str(key) not in self.h5["tiles"].keys(): |
|
251 | 250 | raise KeyError(f"key {key} is not in Tiles") |
|
252 | 251 | del self.h5["tiles"][str(key)] |
@@ -270,7 +269,7 @@
Loading
270 | 269 | raise ValueError( |
|
271 | 270 | f"key {key} already exists in 'masks'. Cannot add. Must update to modify existing mask." |
|
272 | 271 | ) |
|
273 | - | newmask = self.h5["masks"].create_dataset(key, data=mask) |
|
272 | + | self.h5["masks"].create_dataset(key, data=mask) |
|
274 | 273 | ||
275 | 274 | def update_mask(self, key, mask): |
|
276 | 275 | """ |
@@ -340,7 +339,7 @@
Loading
340 | 339 | f"masks keys must be of type(str) but key was passed of type {type(key)}" |
|
341 | 340 | ) |
|
342 | 341 | if key not in self.h5["masks"].keys(): |
|
343 | - | raise KeyError(f"key is not in Masks") |
|
342 | + | raise KeyError("key is not in Masks") |
|
344 | 343 | del self.h5["masks"][key] |
|
345 | 344 | ||
346 | 345 | def get_slidetype(self): |
@@ -3,20 +3,19 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | - | import torch |
|
7 | - | from torch import nn |
|
8 | - | from torch.nn import functional as F |
|
9 | - | import numpy as np |
|
10 | 6 | import cv2 |
|
11 | - | from loguru import logger |
|
12 | - | from skimage.segmentation import watershed |
|
13 | - | from scipy.ndimage.morphology import binary_fill_holes |
|
14 | 7 | import matplotlib.pyplot as plt |
|
8 | + | import numpy as np |
|
9 | + | import torch |
|
10 | + | from loguru import logger |
|
15 | 11 | from matplotlib.colors import TABLEAU_COLORS |
|
16 | - | from warnings import warn |
|
12 | + | from scipy.ndimage.morphology import binary_fill_holes |
|
13 | + | from skimage.segmentation import watershed |
|
14 | + | from torch import nn |
|
15 | + | from torch.nn import functional as F |
|
17 | 16 | ||
18 | - | from pathml.utils import segmentation_lines |
|
19 | 17 | from pathml.ml.utils import center_crop_im_batch, dice_loss, get_sobel_kernels |
|
18 | + | from pathml.utils import segmentation_lines |
|
20 | 19 | ||
21 | 20 | ||
22 | 21 | class _BatchNormRelu(nn.Module): |
@@ -431,7 +430,8 @@
Loading
431 | 430 | ||
432 | 431 | try: |
|
433 | 432 | inst_list.remove(0) # 0 is background |
|
434 | - | except: |
|
433 | + | # TODO: change to specific exception |
|
434 | + | except Exception: |
|
435 | 435 | logger.warning( |
|
436 | 436 | "No pixels with 0 label. This means that there are no background pixels. This may indicate a problem. Ignore this warning if this is expected/intended." |
|
437 | 437 | ) |
@@ -5,13 +5,14 @@
Loading
5 | 5 | ||
6 | 6 | import hashlib |
|
7 | 7 | import os |
|
8 | - | from loguru import logger |
|
9 | 8 | from pathlib import Path |
|
10 | 9 | ||
11 | 10 | import h5py |
|
11 | + | from loguru import logger |
|
12 | + | from torch.utils.data import DataLoader, Dataset |
|
13 | + | ||
12 | 14 | from pathml.datasets.base_data_module import BaseDataModule |
|
13 | 15 | from pathml.utils import download_from_url |
|
14 | - | from torch.utils.data import DataLoader, Dataset |
|
15 | 16 | ||
16 | 17 | ||
17 | 18 | class DeepFocusDataModule(BaseDataModule): |
@@ -47,7 +48,7 @@
Loading
47 | 48 | else: |
|
48 | 49 | assert ( |
|
49 | 50 | self._check_integrity() |
|
50 | - | ), f"download is False but data directory does not exist or md5 checksum failed" |
|
51 | + | ), "download is False but data directory does not exist or md5 checksum failed" |
|
51 | 52 | self.shuffle = shuffle |
|
52 | 53 | self.transforms = transforms |
|
53 | 54 | self.batch_size = batch_size |
@@ -3,16 +3,11 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | - | import ast |
|
7 | 6 | import tempfile |
|
8 | - | from collections import OrderedDict |
|
9 | - | from dataclasses import asdict |
|
10 | - | from loguru import logger |
|
7 | + | ||
11 | 8 | import anndata |
|
12 | 9 | import h5py |
|
13 | 10 | import numpy as np |
|
14 | - | import pathml.core.slide_backends |
|
15 | - | import pathml.core.slide_data |
|
16 | 11 | ||
17 | 12 | ||
18 | 13 | # TODO: Fletcher32 checksum? |
@@ -25,7 +20,7 @@
Loading
25 | 20 | name(str): name of dataset to be created |
|
26 | 21 | df(pd.DataFrame): dataframe to be written |
|
27 | 22 | """ |
|
28 | - | dataset = h5.create_dataset( |
|
23 | + | h5.create_dataset( |
|
29 | 24 | str(name), |
|
30 | 25 | data=df, |
|
31 | 26 | chunks=True, |
@@ -4,7 +4,6 @@
Loading
4 | 4 | """ |
|
5 | 5 | ||
6 | 6 | import pickle |
|
7 | - | from loguru import logger |
|
8 | 7 | ||
9 | 8 | import pathml.core.tile |
|
10 | 9 | from pathml.preprocessing.transforms import Transform |
@@ -22,7 +21,7 @@
Loading
22 | 21 | def __init__(self, transform_sequence=None): |
|
23 | 22 | assert transform_sequence is None or all( |
|
24 | 23 | [isinstance(t, Transform) for t in transform_sequence] |
|
25 | - | ), (f"All elements in input list must be of" f" type pathml.core.Transform") |
|
24 | + | ), "All elements in input list must be of type pathml.core.Transform" |
|
26 | 25 | self.transforms = transform_sequence |
|
27 | 26 | ||
28 | 27 | def __len__(self): |
@@ -32,7 +31,7 @@
Loading
32 | 31 | if self.transforms is None: |
|
33 | 32 | return "Pipeline()" |
|
34 | 33 | else: |
|
35 | - | out = f"Pipeline([\n" |
|
34 | + | out = "Pipeline([\n" |
|
36 | 35 | for t in self.transforms: |
|
37 | 36 | out += f"\t{repr(t)},\n" |
|
38 | 37 | out += "])" |
@@ -8,13 +8,13 @@
Loading
8 | 8 | import shutil |
|
9 | 9 | import zipfile |
|
10 | 10 | from pathlib import Path |
|
11 | - | from warnings import warn |
|
12 | 11 | ||
13 | 12 | import cv2 |
|
14 | 13 | import numpy as np |
|
15 | - | from loguru import logger |
|
16 | 14 | import torch |
|
17 | 15 | import torch.utils.data as data |
|
16 | + | from loguru import logger |
|
17 | + | ||
18 | 18 | from pathml.datasets.base_data_module import BaseDataModule |
|
19 | 19 | from pathml.datasets.utils import pannuke_multiclass_mask_to_nucleus_mask |
|
20 | 20 | from pathml.ml.hovernet import compute_hv_map |
@@ -3,13 +3,13 @@
Loading
3 | 3 | License: GNU GPL 2.0 |
|
4 | 4 | """ |
|
5 | 5 | ||
6 | - | import numpy as np |
|
7 | - | import anndata |
|
6 | + | import reprlib |
|
8 | 7 | from collections import OrderedDict |
|
9 | - | import matplotlib.pyplot as plt |
|
8 | + | ||
9 | + | import anndata |
|
10 | 10 | import h5py |
|
11 | - | import reprlib |
|
12 | - | from loguru import logger |
|
11 | + | import matplotlib.pyplot as plt |
|
12 | + | import numpy as np |
|
13 | 13 | ||
14 | 14 | import pathml.core.masks |
|
15 | 15 |
@@ -153,7 +153,7 @@
Loading
153 | 153 | if self.counts: |
|
154 | 154 | out.append(f"counts matrix of shape {self.counts.shape}") |
|
155 | 155 | else: |
|
156 | - | out.append(f"counts=None") |
|
156 | + | out.append("counts=None") |
|
157 | 157 | out = ",\n\t".join(out) |
|
158 | 158 | out += ")" |
|
159 | 159 | return out |
Files | Coverage |
---|---|
pathml | 86.18% |
Project Totals (25 files) | 86.18% |
codecov-umbrella
PYTHON=undefined OS=undefined
codecov-umbrella
PYTHON=undefined OS=undefined
Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file.
The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files.
The size and color of each slice is representing the number of statements and the coverage, respectively.