#!/usr/bin/python3 #coding=utf-8 import argparse import os import time import numpy as np import shutil import sys from tqdm import tqdm import json from python_tree import color_print_dir import random import torch import pandas import lmdb # from trans_dicom_2_nii import MyEncoder from anno_json_wrapper import annojson2cls, annolabels2labelidlabelmap, CLS_UNDEFINED_LABEL from k8s_utils import CK8sPathWrapper, CParamFiller, CDatasetWrapper, get_fullfilepaths_according2pattern parser = argparse.ArgumentParser(description='PyTorch DataBowl3 Detector') parser.add_argument('--job_data_root', default="/root/Documents/GroupLung/Datasets/cls_data/job_data_cls", type=str) args = parser.parse_args() g_pathWrapper = CK8sPathWrapper(args.job_data_root) config_path = g_pathWrapper.get_input_inputconfig_filepath() if not os.path.isfile(config_path): print(fr'given config file {config_path} not exist !') else: # print(fr'using config file {config_path} to fill args !') args = CParamFiller.fillin_args(config_path, args) import logging import sys from utils import constants as constants def main(): global args g_rank = 0 g_world_size = 1 g_local_rank = 0 save_path = g_pathWrapper.get_output_preprocess_preprocessfiledetails_dirpath() # preprocess结果目录 print(fr'doing prepare, save path {save_path}') _ensure_path(save_path) fulldicom_path = fr'{g_pathWrapper.get_output_tmp_dirpath()}' _ensure_path(fulldicom_path) fullnpy_path = fr'{save_path}/{constants.SUBDIR_NPY}' _ensure_path(fullnpy_path) ############################################ # ================================== print('using mlops data') data_config_json = g_pathWrapper.get_input_dataset_filepath() ## data_set.json print(fr'target data config {data_config_json}') # 将zip解压缩,dcm保存在tmp中,[uid,dicom,json] datainfo_list = CDatasetWrapper.get_dataset_list(data_config_json, g_pathWrapper.get_output_tmp_dirpath()) print(fr'loading datameta length {len(datainfo_list)}') print(fr'done...') print(fr'preprocessing...') # preprocess_LUNGMLOPs(input_path=fulldicom_path, save_path=fullnpy_path, # , diameter_th=args.diameter_th, given_label_id=args.working_label, # datainfos=datainfo_list, log_func=print, # preprocess_workers=args.preprocess_workers) from trans_dicom_2_npy import Trans_Dicom_2_NPY_simple # deal with dcms crop_size=48 trans = Trans_Dicom_2_NPY_simple(fulldicom_path, fullnpy_path, crop_size, datainfo_list) trans() print(fr'done...') def _ensure_path(given_path): if not os.path.exists(given_path): os.makedirs(given_path) def _ensure_params(param, giventype=int): return param if type(param) == giventype else eval(param) if __name__ == '__main__': main() exit() try: status = main() except Exception as ex: print(fr'exception occured, msg {ex}') status = -1 sys.exit(status)