Mercurial > repos > rv43 > tomo

comparison workflow/models.py @ 71:1cf15b61cd83 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://github.com/rolfverberg/galaxytools commit 366e516aef0735af2998c6ff3af037181c8d5213
author rv43
date Mon, 20 Mar 2023 13:56:57 +0000
parents fba792d5f83b
children
comparison
equal deleted inserted replaced
70:97c4e2cbbad9 71:1cf15b61cd83
9 import os 9 import os
10 import yaml 10 import yaml
11 11
12 from functools import cache 12 from functools import cache
13 from pathlib import PosixPath 13 from pathlib import PosixPath
14 from pydantic import validator, ValidationError, conint, confloat, constr, \
15 conlist, FilePath, PrivateAttr
16 from pydantic import BaseModel as PydanticBaseModel 14 from pydantic import BaseModel as PydanticBaseModel
15 from pydantic import validator, ValidationError, conint, confloat, constr, conlist, FilePath, \
16 PrivateAttr
17 from nexusformat.nexus import * 17 from nexusformat.nexus import *
18 from time import time 18 from time import time
19 from typing import Optional, Literal 19 from typing import Optional, Literal
20 from typing_extensions import TypedDict 20 from typing_extensions import TypedDict
21 try: 21 try:
22 from pyspec.file.spec import FileSpec 22 from pyspec.file.spec import FileSpec
23 except: 23 except:
24 pass 24 pass
25 25
26 from msnctools.general import is_int, is_num, input_int, input_int_list, input_num, input_yesno, \ 26 try:
27 input_menu, index_nearest, string_to_list, file_exists_and_readable 27 from msnctools.general import is_int, is_num, input_int, input_int_list, input_num, \
28 input_yesno, input_menu, index_nearest, string_to_list, file_exists_and_readable
29 except:
30 from general import is_int, is_num, input_int, input_int_list, input_num, \
31 input_yesno, input_menu, index_nearest, string_to_list, file_exists_and_readable
28 32
29 33
30 def import_scanparser(station): 34 def import_scanparser(station):
31 if station in ('id1a3', 'id3a'): 35 if station in ('id1a3', 'id3a'):
32 from msnctools.scanparsers import SMBRotationScanParser 36 try:
33 globals()['ScanParser'] = SMBRotationScanParser 37 from msnctools.scanparsers import SMBRotationScanParser
38 globals()['ScanParser'] = SMBRotationScanParser
39 except:
40 try:
41 from scanparsers import SMBRotationScanParser
42 globals()['ScanParser'] = SMBRotationScanParser
43 except:
44 pass
34 elif station in ('id3b'): 45 elif station in ('id3b'):
35 from msnctools.scanparsers import FMBRotationScanParser 46 try:
36 globals()['ScanParser'] = FMBRotationScanParser 47 from msnctools.scanparsers import FMBRotationScanParser
48 globals()['ScanParser'] = FMBRotationScanParser
49 except:
50 try:
51 from scanparsers import FMBRotationScanParser
52 globals()['ScanParser'] = FMBRotationScanParser
53 except:
54 pass
37 else: 55 else:
38 raise RuntimeError(f'Invalid station: {station}') 56 raise RuntimeError(f'Invalid station: {station}')
39 57
40 @cache 58 @cache
41 def get_available_scan_numbers(spec_file:str): 59 def get_available_scan_numbers(spec_file:str):
43 scan_numbers = list(scans.keys()) 61 scan_numbers = list(scans.keys())
44 for scan_number in scan_numbers.copy(): 62 for scan_number in scan_numbers.copy():
45 try: 63 try:
46 parser = ScanParser(spec_file, scan_number) 64 parser = ScanParser(spec_file, scan_number)
47 try: 65 try:
48 scan_type = parser.get_scan_type() 66 scan_type = parser.scan_type
49 except: 67 except:
50 scan_type = None 68 scan_type = None
51 pass
52 except: 69 except:
53 scan_numbers.remove(scan_number) 70 scan_numbers.remove(scan_number)
54 return(scan_numbers) 71 return(scan_numbers)
55 72
56 @cache 73 @cache
432 return(None) 449 return(None)
433 450
434 def get_scanparser(self, scan_number): 451 def get_scanparser(self, scan_number):
435 return(get_scanparser(self.spec_file, scan_number)) 452 return(get_scanparser(self.spec_file, scan_number))
436 453
437 # def get_detector_data(self, detector_prefix, scan_number=None, scan_step_index=None):
438 # if scan_number is None:
439 # scan_number = self.scan_numbers[0]
440 # if scan_step_index is None:
441 # scan_info = self.stack_info[self.get_scan_index(scan_number)]
442 # scan_step_index = scan_info['starting_image_offset']
443 # parser = self.get_scanparser(scan_number)
444 # return(parser.get_detector_data(detector_prefix, scan_step_index))
445
446 def get_detector_data(self, detector_prefix, scan_number=None, scan_step_index=None): 454 def get_detector_data(self, detector_prefix, scan_number=None, scan_step_index=None):
447 image_stacks = [] 455 image_stacks = []
448 if scan_number is None: 456 if scan_number is None:
449 scan_numbers = self.scan_numbers 457 scan_numbers = self.scan_numbers
450 else: 458 else:
458 image_stacks.append(parser.get_detector_data(detector_prefix, 466 image_stacks.append(parser.get_detector_data(detector_prefix,
459 (image_offset, image_offset+num_image))) 467 (image_offset, image_offset+num_image)))
460 else: 468 else:
461 image_stacks.append(parser.get_detector_data(detector_prefix, 469 image_stacks.append(parser.get_detector_data(detector_prefix,
462 image_offset+scan_step_index)) 470 image_offset+scan_step_index))
463 if len(image_stacks) == 1: 471 if scan_number is not None and scan_step_index is not None:
472 # Return a single image for a specific scan_number and scan_step_index request
464 return(image_stacks[0]) 473 return(image_stacks[0])
465 else: 474 else:
475 # Return a list otherwise
466 return(image_stacks) 476 return(image_stacks)
477 return(image_stacks)
467 478
468 def scan_numbers_cli(self, attr_desc, **kwargs): 479 def scan_numbers_cli(self, attr_desc, **kwargs):
469 available_scan_numbers = self.available_scan_numbers 480 available_scan_numbers = self.available_scan_numbers
470 station = kwargs.get('station') 481 station = kwargs.get('station')
471 if (station is not None and station in ('id1a3', 'id3a') and 482 if (station is not None and station in ('id1a3', 'id3a') and
473 scan_type = kwargs['scan_type'] 484 scan_type = kwargs['scan_type']
474 if scan_type == 'ts1': 485 if scan_type == 'ts1':
475 available_scan_numbers = [] 486 available_scan_numbers = []
476 for scan_number in self.available_scan_numbers: 487 for scan_number in self.available_scan_numbers:
477 parser = self.get_scanparser(scan_number) 488 parser = self.get_scanparser(scan_number)
478 if parser.scan_type == scan_type: 489 try:
479 available_scan_numbers.append(scan_number) 490 if parser.scan_type == scan_type:
491 available_scan_numbers.append(scan_number)
492 except:
493 pass
480 elif scan_type == 'df1': 494 elif scan_type == 'df1':
481 tomo_scan_numbers = kwargs['tomo_scan_numbers'] 495 tomo_scan_numbers = kwargs['tomo_scan_numbers']
482 available_scan_numbers = [] 496 available_scan_numbers = []
483 for scan_number in tomo_scan_numbers: 497 for scan_number in tomo_scan_numbers:
484 parser = self.get_scanparser(scan_number-2) 498 parser = self.get_scanparser(scan_number-2)
585 image_offset = parser.starting_image_offset 599 image_offset = parser.starting_image_offset
586 num_image = parser.get_num_image(detector_prefix.upper()) 600 num_image = parser.get_num_image(detector_prefix.upper())
587 scan_index = self.get_scan_index(scan_number) 601 scan_index = self.get_scan_index(scan_number)
588 602
589 # Select the image set 603 # Select the image set
590 last_image_index = image_offset+num_image-1 604 last_image_index = image_offset+num_image
591 print(f'Available good image set index range: [{image_offset}, {last_image_index}]') 605 print(f'Available good image set index range: [{image_offset}, {last_image_index})')
592 image_set_approved = False 606 image_set_approved = False
593 if scan_index is not None: 607 if scan_index is not None:
594 scan_info = stack_info[scan_index] 608 scan_info = stack_info[scan_index]
595 print(f'Current starting image offset and number of images: '+ 609 print(f'Current starting image offset and number of images: '+
596 f'{scan_info["starting_image_offset"]} and {scan_info["num_image"]}') 610 f'{scan_info["starting_image_offset"]} and {scan_info["num_image"]}')
597 image_set_approved = input_yesno(f'Accept these values (y/n)?', 'y') 611 image_set_approved = input_yesno(f'Accept these values (y/n)?', 'y')
598 if not image_set_approved: 612 if not image_set_approved:
599 print(f'Default starting image offset and number of images: '+ 613 print(f'Default starting image offset and number of images: '+
600 f'{image_offset} and {last_image_index-image_offset}') 614 f'{image_offset} and {num_image}')
601 image_set_approved = input_yesno(f'Accept these values (y/n)?', 'y') 615 image_set_approved = input_yesno(f'Accept these values (y/n)?', 'y')
602 if image_set_approved: 616 if image_set_approved:
603 offset = image_offset 617 offset = image_offset
604 num = last_image_index-offset 618 num = last_image_index-offset
605 while not image_set_approved: 619 while not image_set_approved:
606 offset = input_int(f'Enter the starting image offset', ge=image_offset, 620 offset = input_int(f'Enter the starting image offset', ge=image_offset,
607 le=last_image_index-1)#, default=image_offset) 621 lt=last_image_index)#, default=image_offset)
608 num = input_int(f'Enter the number of images', ge=1, 622 num = input_int(f'Enter the number of images', ge=1,
609 le=last_image_index-offset+1)#, default=last_image_index-offset+1) 623 le=last_image_index-offset)#, default=last_image_index-offset)
610 print(f'Current starting image offset and number of images: {offset} and {num}') 624 print(f'Current starting image offset and number of images: {offset} and {num}')
611 image_set_approved = input_yesno(f'Accept these values (y/n)?', 'y') 625 image_set_approved = input_yesno(f'Accept these values (y/n)?', 'y')
612 if scan_index is not None: 626 if scan_index is not None:
613 scan_info['starting_image_offset'] = offset 627 scan_info['starting_image_offset'] = offset
614 scan_info['num_image'] = num 628 scan_info['num_image'] = num
686 scan_numbers = self.scan_numbers 700 scan_numbers = self.scan_numbers
687 else: 701 else:
688 scan_numbers = [scan_number] 702 scan_numbers = [scan_number]
689 for scan_number in scan_numbers: 703 for scan_number in scan_numbers:
690 parser = self.get_scanparser(scan_number) 704 parser = self.get_scanparser(scan_number)
691 horizontal_shifts.append(parser.get_horizontal_shift()) 705 horizontal_shifts.append(parser.horizontal_shift)
692 if len(horizontal_shifts) == 1: 706 if len(horizontal_shifts) == 1:
693 return(horizontal_shifts[0]) 707 return(horizontal_shifts[0])
694 else: 708 else:
695 return(horizontal_shifts) 709 return(horizontal_shifts)
696 710
700 scan_numbers = self.scan_numbers 714 scan_numbers = self.scan_numbers
701 else: 715 else:
702 scan_numbers = [scan_number] 716 scan_numbers = [scan_number]
703 for scan_number in scan_numbers: 717 for scan_number in scan_numbers:
704 parser = self.get_scanparser(scan_number) 718 parser = self.get_scanparser(scan_number)
705 vertical_shifts.append(parser.get_vertical_shift()) 719 vertical_shifts.append(parser.vertical_shift)
706 if len(vertical_shifts) == 1: 720 if len(vertical_shifts) == 1:
707 return(vertical_shifts[0]) 721 return(vertical_shifts[0])
708 else: 722 else:
709 return(vertical_shifts) 723 return(vertical_shifts)
710 724
726 f'\n\tScan {scan_number}: {theta_vals}'+ 740 f'\n\tScan {scan_number}: {theta_vals}'+
727 f'\n\tScan {self.scan_numbers[0]}: {parser.theta_vals}') 741 f'\n\tScan {self.scan_numbers[0]}: {parser.theta_vals}')
728 return 742 return
729 743
730 # Select the theta range for the tomo reconstruction from the first scan 744 # Select the theta range for the tomo reconstruction from the first scan
745 theta_range_approved = False
731 thetas = np.linspace(spec_theta_start, spec_theta_end, spec_num_theta) 746 thetas = np.linspace(spec_theta_start, spec_theta_end, spec_num_theta)
732 delta_theta = thetas[1]-thetas[0] 747 delta_theta = thetas[1]-thetas[0]
733 theta_range_approved = False 748 print(f'Theta range obtained from SPEC data: [{spec_theta_start}, {spec_theta_end}]')
734 print(f'Theta range obtained from SPEC data: [{spec_theta_start}, {spec_theta_end})')
735 print(f'Theta step size = {delta_theta}') 749 print(f'Theta step size = {delta_theta}')
736 print(f'Number of theta values: {spec_num_theta}') 750 print(f'Number of theta values: {spec_num_theta}')
737 exit('Done')
738 default_start = None 751 default_start = None
739 default_end = None 752 default_end = None
740 if station in ('id1a3', 'id3a'): 753 if station in ('id1a3', 'id3a'):
741 theta_range_approved = input_yesno(f'Accept this theta range (y/n)?', 'y') 754 theta_range_approved = input_yesno(f'Accept this theta range (y/n)?', 'y')
742 if theta_range_approved: 755 if theta_range_approved:
743 theta_start = spec_theta_start 756 self.theta_range = {'start': float(spec_theta_start), 'end': float(spec_theta_end),
744 theta_end = spec_theta_end 757 'num': int(spec_num_theta), 'start_index': 0}
745 num_theta = spec_num_theta 758 return
746 theta_index_start = 0
747 elif station in ('id3b'): 759 elif station in ('id3b'):
748 if spec_theta_start <= 0.0 and spec_theta_end >= 180.0: 760 if spec_theta_start <= 0.0 and spec_theta_end >= 180.0:
749 default_start = 0 761 default_start = 0
750 default_end = 180 762 default_end = 180
751 elif spec_theta_end-spec_theta_start == 180: 763 elif spec_theta_end-spec_theta_start == 180:
763 num_theta = theta_index_end-theta_index_start 775 num_theta = theta_index_end-theta_index_start
764 print(f'Selected theta range: [{theta_start}, {theta_start+delta_theta}, ..., '+ 776 print(f'Selected theta range: [{theta_start}, {theta_start+delta_theta}, ..., '+
765 f'{theta_end})') 777 f'{theta_end})')
766 print(f'Number of theta values: {num_theta}') 778 print(f'Number of theta values: {num_theta}')
767 theta_range_approved = input_yesno(f'Accept this theta range (y/n)?', 'y') 779 theta_range_approved = input_yesno(f'Accept this theta range (y/n)?', 'y')
768 self.thetas = np.linspace(theta_start, theta_end, num_theta) 780 self.theta_range = {'start': float(theta_start), 'end': float(theta_end),
781 'num': int(num_theta), 'start_index': int(theta_index_start)}
769 782
770 def image_range_cli(self, attr_desc, detector_prefix): 783 def image_range_cli(self, attr_desc, detector_prefix):
771 stack_info = self.stack_info 784 stack_info = self.stack_info
772 for scan_number in self.scan_numbers: 785 for scan_number in self.scan_numbers:
773 # Parse the available image range 786 # Parse the available image range
803 attr_desc = '' 816 attr_desc = ''
804 cycle = cli_kwargs.get('cycle') 817 cycle = cli_kwargs.get('cycle')
805 btr = cli_kwargs.get('btr') 818 btr = cli_kwargs.get('btr')
806 station = cli_kwargs.get('station') 819 station = cli_kwargs.get('station')
807 detector = cli_kwargs.get('detector') 820 detector = cli_kwargs.get('detector')
821 sample_name = cli_kwargs.get('sample_name')
808 print(f'\n -- Configure the location of the {attr_desc}scan data -- ') 822 print(f'\n -- Configure the location of the {attr_desc}scan data -- ')
809 if station in ('id1a3', 'id3a'): 823 if station in ('id1a3', 'id3a'):
810 basedir = f'/nfs/chess/{station}/{cycle}/{btr}' 824 basedir = f'/nfs/chess/{station}/{cycle}/{btr}'
811 runs = [d for d in os.listdir(basedir) if os.path.isdir(os.path.join(basedir, d))] 825 runs = [d for d in os.listdir(basedir) if os.path.isdir(os.path.join(basedir, d))]
812 #RV index = 15-1 826 #RV index = 15-1
813 #RV index = 7-1 827 #RV index = 7-1
814 index = input_menu(runs, header='Choose a sample directory') 828 if sample_name is not None and sample_name in runs:
829 index = runs.index(sample_name)
830 else:
831 index = input_menu(runs, header='Choose a sample directory')
815 self.spec_file = f'{basedir}/{runs[index]}/spec.log' 832 self.spec_file = f'{basedir}/{runs[index]}/spec.log'
816 self.scan_numbers_cli(attr_desc, station=station, scan_type='ts1') 833 self.scan_numbers_cli(attr_desc, station=station, scan_type='ts1')
817 else: 834 else:
818 self.set_single_attr_cli('spec_file', attr_desc+'SPEC file path') 835 self.set_single_attr_cli('spec_file', attr_desc+'SPEC file path')
819 self.scan_numbers_cli(attr_desc) 836 self.scan_numbers_cli(attr_desc)
843 config['z_translation'] = nxsample.z_translation.nxdata 860 config['z_translation'] = nxsample.z_translation.nxdata
844 return(cls(**config)) 861 return(cls(**config))
845 862
846 def cli(self): 863 def cli(self):
847 print('\n -- Configure the sample metadata -- ') 864 print('\n -- Configure the sample metadata -- ')
848 #RV self.name = 'test'
849 #RV self.name = 'sobhani-3249-A' 865 #RV self.name = 'sobhani-3249-A'
866 #RV self.name = 'tenstom_1304r-1'
850 self.set_single_attr_cli('name', 'the sample name') 867 self.set_single_attr_cli('name', 'the sample name')
851 #RV self.description = 'test sample' 868 #RV self.description = 'test sample'
852 self.set_single_attr_cli('description', 'a description of the sample (optional)') 869 self.set_single_attr_cli('description', 'a description of the sample (optional)')
853 870
854 871
915 use_detector_config = False 932 use_detector_config = False
916 if hasattr(self.detector, 'prefix') and len(self.detector.prefix): 933 if hasattr(self.detector, 'prefix') and len(self.detector.prefix):
917 use_detector_config = input_yesno(f'Current detector settings:\n{self.detector}\n'+ 934 use_detector_config = input_yesno(f'Current detector settings:\n{self.detector}\n'+
918 f'Keep these settings? (y/n)') 935 f'Keep these settings? (y/n)')
919 if not use_detector_config: 936 if not use_detector_config:
920 #RV have_detector_config = True 937 menu_options = ['not listed', 'andor2', 'manta', 'retiga']
921 have_detector_config = input_yesno(f'Is a detector configuration file available? (y/n)') 938 input_mode = input_menu(menu_options, header='Choose one of the following detector '+
922 if have_detector_config: 939 'configuration options')
923 #RV detector_config_file = 'retiga.yaml' 940 if input_mode:
924 #RV detector_config_file = 'andor2.yaml' 941 detector_config_file = f'{menu_options[input_mode]}.yaml'
925 detector_config_file = input(f'Enter detector configuration file name: ')
926 have_detector_config = self.detector.construct_from_yaml(detector_config_file) 942 have_detector_config = self.detector.construct_from_yaml(detector_config_file)
943 else:
944 have_detector_config = False
927 if not have_detector_config: 945 if not have_detector_config:
928 self.set_single_attr_cli('detector', 'detector') 946 self.set_single_attr_cli('detector', 'detector')
929 self.set_single_attr_cli('tomo_fields', 'Tomo field', chain_attr_desc=True, 947 self.set_single_attr_cli('tomo_fields', 'Tomo field', chain_attr_desc=True,
930 cycle=self.cycle, btr=self.btr, station=self.station, detector=self.detector) 948 cycle=self.cycle, btr=self.btr, station=self.station, detector=self.detector,
949 sample_name=self.sample.name)
931 if self.station in ('id1a3', 'id3a'): 950 if self.station in ('id1a3', 'id3a'):
932 have_dark_field = True 951 have_dark_field = True
933 tomo_spec_file = self.tomo_fields.spec_file 952 tomo_spec_file = self.tomo_fields.spec_file
934 else: 953 else:
935 have_dark_field = input_yesno(f'Are Dark field images available? (y/n)') 954 have_dark_field = input_yesno(f'Are Dark field images available? (y/n)')
999 thetas = None 1018 thetas = None
1000 # Add the scans in a single spec file 1019 # Add the scans in a single spec file
1001 nxspec_scans[field_name] = field.construct_nxcollection(image_key, thetas, 1020 nxspec_scans[field_name] = field.construct_nxcollection(image_key, thetas,
1002 self.detector) 1021 self.detector)
1003 if include_raw_data: 1022 if include_raw_data:
1004 image_stacks.append(field.get_detector_data(self.detector.prefix)) 1023 image_stacks += field.get_detector_data(self.detector.prefix)
1005 for scan_number in field.scan_numbers: 1024 for scan_number in field.scan_numbers:
1006 parser = field.get_scanparser(scan_number) 1025 parser = field.get_scanparser(scan_number)
1007 scan_info = field.stack_info[field.get_scan_index(scan_number)] 1026 scan_info = field.stack_info[field.get_scan_index(scan_number)]
1008 num_image = scan_info['num_image'] 1027 num_image = scan_info['num_image']
1009 image_keys += num_image*[image_key] 1028 image_keys += num_image*[image_key]