
    .hL                        d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlZd dlZd dlmZ d dlmZmZmZmZ d d	lmZmZmZmZmZ d d
lmZ  G d de      Z y)    )annotationsN)deepcopy)
ThreadPool)Path)Any)Dataset)FORMATS_HELP_MSGHELP_URLIMG_FORMATScheck_file_speeds)DEFAULT_CFG
LOCAL_RANKLOGGERNUM_THREADSTQDM)imreadc                       e Zd ZdZdddeddddddd	d
df	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZddZddZdddZd dZ	d!dZ
d"d#dZd"d#dZd dZd$dZd$dZd%dZd&dZd'd(dZd)dZ xZS )*BaseDataseta
  
    Base dataset class for loading and processing image data.

    This class provides core functionality for loading images, caching, and preparing data for training and inference
    in object detection tasks.

    Attributes:
        img_path (str): Path to the folder containing images.
        imgsz (int): Target image size for resizing.
        augment (bool): Whether to apply data augmentation.
        single_cls (bool): Whether to treat all objects as a single class.
        prefix (str): Prefix to print in log messages.
        fraction (float): Fraction of dataset to utilize.
        channels (int): Number of channels in the images (1 for grayscale, 3 for RGB).
        cv2_flag (int): OpenCV flag for reading images.
        im_files (list[str]): List of image file paths.
        labels (list[dict]): List of label data dictionaries.
        ni (int): Number of images in the dataset.
        rect (bool): Whether to use rectangular training.
        batch_size (int): Size of batches.
        stride (int): Stride used in the model.
        pad (float): Padding value.
        buffer (list): Buffer for mosaic images.
        max_buffer_length (int): Maximum buffer size.
        ims (list): List of loaded images.
        im_hw0 (list): List of original image dimensions (h, w).
        im_hw (list): List of resized image dimensions (h, w).
        npy_files (list[Path]): List of numpy file paths.
        cache (str): Cache images to RAM or disk during training.
        transforms (callable): Image transformation function.
        batch_shapes (np.ndarray): Batch shapes for rectangular training.
        batch (np.ndarray): Batch index of each image.

    Methods:
        get_img_files: Read image files from the specified path.
        update_labels: Update labels to include only specified classes.
        load_image: Load an image from the dataset.
        cache_images: Cache images to memory or disk.
        cache_images_to_disk: Save an image as an *.npy file for faster loading.
        check_cache_disk: Check image caching requirements vs available disk space.
        check_cache_ram: Check image caching requirements vs available memory.
        set_rectangle: Set the shape of bounding boxes as rectangles.
        get_image_and_label: Get and return label information from the dataset.
        update_labels_info: Custom label format method to be implemented by subclasses.
        build_transforms: Build transformation pipeline to be implemented by subclasses.
        get_labels: Get labels method to be implemented by subclasses.
    i  FT              ?Ng      ?   c                $   t         |           || _        || _        || _        || _        || _        || _        || _        |dk(  rt        j                  nt        j                  | _        | j                  | j                        | _        | j                         | _        | j#                  |       t%        | j                         | _        || _        || _        |	| _        |
| _        | j(                  r| j*                  J | j1                          g | _        | j                  r%t5        | j&                  | j*                  dz  df      nd| _        dg| j&                  z  dg| j&                  z  dg| j&                  z  c| _        | _        | _        | j                  D cg c]  }t?        |      jA                  d       c}| _!        tE        |tF              r|jI                         n|du rd	nd| _%        | jJ                  d	k(  rB| jM                         r2|jN                  rtQ        jR                  d
       | jU                          n/| jJ                  dk(  r | jW                         r| jU                          | jY                  |      | _-        yc c}w )a  
        Initialize BaseDataset with given configuration and options.

        Args:
            img_path (str | list[str]): Path to the folder containing images or list of image paths.
            imgsz (int): Image size for resizing.
            cache (bool | str): Cache images to RAM or disk during training.
            augment (bool): If True, data augmentation is applied.
            hyp (dict[str, Any]): Hyperparameters to apply data augmentation.
            prefix (str): Prefix to print in log messages.
            rect (bool): If True, rectangular training is used.
            batch_size (int): Size of batches.
            stride (int): Stride used in the model.
            pad (float): Padding value.
            single_cls (bool): If True, single class training is used.
            classes (list[int], optional): List of included classes.
            fraction (float): Fraction of dataset to utilize.
            channels (int): Number of channels in the images (1 for grayscale, 3 for RGB).
           )include_classN   i  r   z.npyTramzcache='ram' may produce non-deterministic training results. Consider cache='disk' as a deterministic alternative if your disk space allows.disk)hyp).super__init__img_pathimgszaugment
single_clsprefixfractionchannelscv2IMREAD_GRAYSCALEIMREAD_COLORcv2_flagget_img_filesim_files
get_labelslabelsupdate_labelslennirect
batch_sizestridepadset_rectanglebufferminmax_buffer_lengthimsim_hw0im_hwr   with_suffix	npy_files
isinstancestrlowercachecheck_cache_ramdeterministicr   warningcache_imagescheck_cache_diskbuild_transforms
transforms)selfr#   r$   rE   r%   r    r'   r5   r6   r7   r8   r&   classesr(   r)   f	__class__s                   S/var/www/html/ai-service/venv/lib/python3.12/site-packages/ultralytics/data/base.pyr"   zBaseDataset.__init__H   s   H 	 
$  08A,,3CSCS**4==9oo'1dkk"	$99??...  NRlldggt/BD%I!J`a .2FTWW,<tftww>NQUPVY]Y`Y`P`)$+tz?C}}M!$q'--f5M&0&<U[[]5TX=%^b
::4#7#7#9  f ZZ6!d&;&;&= //C/8 Ns   !Jc           
        	 g }t        |t              r|n|gD ]  }t        |      }|j                         r*|t	        j                  t        |dz  dz        d      z  }I|j                         rt        |d      5 }|j                         j                         j                         }t        |j                        t        j                  z   }||D cg c]'  }|j                  d      r|j                  d|      n|) c}z  }ddd       t!        | j"                   | d	       t%        d
 |D              }|sJ | j"                   d| dt&                	 | j,                  dk  r$|dt/        t1        |      | j,                  z         }t3        || j"                         |S c c}w # 1 sw Y   xY w# t(        $ r'}t!        | j"                   d| dt*               |d}~ww xY w)aN  
        Read image files from the specified path.

        Args:
            img_path (str | list[str]): Path or list of paths to image directories or files.

        Returns:
            (list[str]): List of image file paths.

        Raises:
            FileNotFoundError: If no images are found or the path doesn't exist.
        z**z*.*T)	recursivezutf-8)encodingz./Nz does not existc              3     K   | ]M  }|j                  d       d   j                         t        v s,|j                  dt        j
                         O yw)./N)
rpartitionrD   r   replaceossep).0xs     rQ   	<genexpr>z,BaseDataset.get_img_files.<locals>.<genexpr>   s@     p1<<PSCTUWCXC^C^C`doCoaiiRVV4ps
   -A%AzNo images found in z. zError loading data from 
r   )r'   )rB   listr   is_dirglobrC   is_fileopenreadstrip
splitlinesparentr[   r\   
startswithrZ   FileNotFoundErrorr'   sortedr	   	Exceptionr
   r(   roundr3   r   )	rM   r#   rO   ptri   r^   r/   es	            rQ   r.   zBaseDataset.get_img_files   s   	kA!+Hd!;X( PG88:3q4x%'7#8DIIAYY[a'2 _aFFHNN,779!$QXX!7\]^WXd9Kaiif5QRR^^_ _ ,t{{mA3o,NOOP pappH^},?zL\K]^^8 ==1 F%H(E"FGH(4;;7 __ _  	k#t{{m3KH:UWX`Wa$bcijj	ksD   A=F< ?AF/,F*
F/AF< *F//F9	4F< <	G,"G''G,c                   t        j                  |      j                  dd      }t        t	        | j
                              D ]  }|| j
                  |   d   }| j
                  |   d   }| j
                  |   d   }| j
                  |   d   }||k(  j                  d      }||   | j
                  |   d<   ||   | j
                  |   d<   |r4t        |      D 	
cg c]  \  }	}
|
s	||	    c}
}	| j
                  |   d<   |||   | j
                  |   d<   | j                  sd| j
                  |   d   dddf<    yc c}
}	w )	z
        Update labels to include only specified classes.

        Args:
            include_class (list[int], optional): List of classes to include. If None, all classes are included.
        r   rW   Nclsbboxessegments	keypointsr   )	nparrayreshaperanger3   r1   any	enumerater&   )rM   r   include_class_arrayirs   rt   ru   rv   jsiidxs              rQ   r2   zBaseDataset.update_labels   sA    !hh}5==aDs4;;'( 	0A(kk!nU+Q1;;q>*5 KKN;7	//44Q7(+AAu%+1!9Ax(NWXYl1b72s^a(2,1bDKKN:.(2;A,DKKN;/./Au%ad+	0 2cs   
E*Ec                ^   | j                   |   | j                  |   | j                  |   }}}|b|j                         r	 t	        j
                  |      }nt        || j                        }|t        d|       |j                  dd \  }}|r| j                   t#        ||      z  }	|	d	k7  rt%        t'        j(                  ||	z        | j                         t%        t'        j(                  ||	z        | j                         }}
t+        j,                  ||
|ft*        j.                  
      }nS||cxk(  r| j                   k(  s>n t+        j,                  || j                   | j                   ft*        j.                  
      }|j0                  dk(  r|d   }| j2                  r|||f|j                  dd c| j                   |<   | j4                  |<   | j6                  |<   | j8                  j;                  |       d	t=        | j8                        cxk  r| j>                  k\  rZn nW| j8                  jA                  d      }| jB                  dk7  r-d\  | j                   |<   | j4                  |<   | j6                  |<   |||f|j                  dd fS | j                   |   | j4                  |   | j6                  |   fS # t        $ rd}t        j                  | j                   d| d|        t        |      j                  d       t        || j                        }Y d}~d}~ww xY w)a  
        Load an image from dataset index 'i'.

        Args:
            i (int): Index of the image to load.
            rect_mode (bool): Whether to use rectangular resizing.

        Returns:
            im (np.ndarray): Loaded image as a NumPy array.
            hw_original (tuple[int, int]): Original image dimensions in (height, width) format.
            hw_resized (tuple[int, int]): Resized image dimensions in (height, width) format.

        Raises:
            FileNotFoundError: If the image file is not found.
        Nz"Removing corrupt *.npy image file z	 due to: T)
missing_ok)flagszImage Not Found    r   )interpolation).Nr   r   )NNN)"r=   r/   rA   existsrw   loadrm   r   rH   r'   r   unlinkr   r-   rk   shaper$   maxr;   mathceilr*   resizeINTER_LINEARndimr%   r>   r?   r:   appendr3   r<   poprE   )rM   r~   	rect_modeimrO   fnrq   h0w0rwhr   s                rQ   
load_imagezBaseDataset.load_image   s     HHQKq!14>>!3DrA:yy{8B AT]]3z'*:1#(>??XXbq\FBJJR,6		"q& 14::>DIIbSTfDUW[WaWa@bqABAc>N>NOBB,$**,ZZTZZ$<CL\L\]ww!|	] ||=?"b288TVUV<:T[[^TZZ]""1%s4;;'A4+A+AA*AzzU*EUBT[[^TZZ]Bx"1--xx{DKKNDJJqM99? ! 8NNdkk]2TUWTXXabcad#efHOOtO47B8s   J? ?	L,AL''L,c                   d\  }}| j                   dk(  r| j                  dfn| j                  df\  }}t        t              5 }|j                  |t        | j                              }t        t        |      | j                  t        dkD        }|D ]  \  }}	| j                   dk(  r+|| j                  |   j                         j                  z  }nI|	\  | j                  |<   | j                  |<   | j                   |<   || j                  |   j"                  z  }| j$                   d||z  dd	| d
|_         |j)                          ddd       y# 1 sw Y   yxY w)z3Cache images to memory or disk for faster training.r   i   @r   DiskRAMr   )totaldisablezCaching images (.1fzGB )N)rE   cache_images_to_diskr   r   r   imaprz   r4   r   r|   r   rA   statst_sizer=   r>   r?   nbytesr'   descclose)
rM   bgbfcnstoragepoolresultspbarr~   r^   s
             rQ   rI   zBaseDataset.cache_images
  s<   2>BjjF>R116:Y]YhYhjoXpW$ 
	iiU477^4G	'*$'':PQ>RD V1::'*//1999AAB>DHHQKQA!+++A#{{m+;AF3<s7)STU	V JJL
	 
	 
	s   D	EEc                    | j                   |   }|j                         s=t        j                  |j	                         t        | j                  |         d       yy)z2Save an image as an *.npy file for faster loading.F)allow_pickleN)rA   r   rw   saveas_posixr   r/   )rM   r~   rO   s      rQ   r   z BaseDataset.cache_images_to_disk  sB    NN1xxzGGAJJL&q)9":O     c                   ddl }d\  }}t        | j                  d      }t        |      D ]  }t	        j
                  | j                        }t        |      }|0||j                  z  }t        j                  t        |      j                  t        j                        rwd| _        t        j                   | j"                   d        y || j                  z  |z  d|z   z  }	|j%                  t        | j                  d         j                        \  }
}}|	|kD  rMd| _        t        j                   | j"                   |	|z  dd	t'        |d
z         d||z  dd|
|z  dd	       yy)a  
        Check if there's enough disk space for caching images.

        Args:
            safety_margin (float): Safety margin factor for disk space calculation.

        Returns:
            (bool): True if there's enough disk space, False otherwise.
        r   Nr      z8Skipping caching images to disk, directory not writeableFr   r   zGB disk space required, with d   % safety margin but only rX   z#GB free, not caching images to diskT)shutilr;   r4   rz   randomchoicer/   r   r   r[   accessr   ri   W_OKrE   r   rH   r'   
disk_usageint)rM   safety_marginr   r   r   n_im_filer   disk_requiredr   usedfrees                rQ   rJ   zBaseDataset.check_cache_disk   sZ    	2q 		AmmDMM2GBzNA99T']11277;!
$++.fgh		 DGGa1}+<="--d4==3C.D.K.KLtT4DJNN;;- 237 8MC/011J"9S/52:c"22UW
 r   c                   d\  }}t        | j                  d      }t        |      D ]u  }t        t	        j
                  | j                              }|.| j                  t        |j                  d   |j                  d         z  }||j                  |dz  z  z  }w || j                  z  |z  d|z   z  }t        d      j                         }	||	j                  kD  rad| _        t        j                   | j"                   ||z  dd	t%        |d
z         d|	j                  |z  dd|	j&                  |z  dd	       yy)z
        Check if there's enough RAM for caching images.

        Args:
            safety_margin (float): Safety margin factor for RAM calculation.

        Returns:
            (bool): True if there's enough RAM, False otherwise.
        r   r   Nr   r   r   psutilr   z%GB RAM required to cache images with r   r   rX   z GB available, not caching imagesFT)r;   r4   rz   r   r   r   r/   r$   r   r   r   
__import__virtual_memory	availablerE   r   rH   r'   r   r   )
rM   r   r   r   r   r   r   ratiomem_requiredmems
             rQ   rF   zBaseDataset.check_cache_ramD  s:    2q 	&Admm45BzJJRXXa["((1+!>>EUAX%%A	& 477{Q!m*;<"113#--'DJNN;;-r 1#6 7MC/011J==2%c*!CIIN3+??_a
 r   c                   t        j                  t        j                  | j                        | j                  z        j                  t              }|d   dz   }t        j                  | j                  D cg c]  }|j                  d       c}      }|dddf   |dddf   z  }|j                         }|D cg c]  }| j                  |    c}| _        |D cg c]  }| j                  |    c}| _        ||   }ddgg|z  }t        |      D ]G  }|||k(     }	|	j                         |	j                         }}
|dk  r|dg||<   8|
dkD  s>dd|
z  g||<   I t        j                  t        j                  |      | j                   z  | j"                  z  | j$                  z         j                  t              | j"                  z  | _        || _        yc c}w c c}w c c}w )zBSet the shape of bounding boxes for YOLO detections as rectangles.rW   r   r   Nr   )rw   flooraranger4   r6   astyper   rx   r1   r   argsortr/   rz   r;   r   r   r$   r7   r8   batch_shapesbatch)rM   binbr^   sarirectr~   shapesariminimaxis               rQ   r9   zBaseDataset.set_rectangleb  s   XXbii(4??:;BB3GVaZHHdkk:aeeGn:;q!tWqAw

389aq)9/45!t{{1~5Y a&Br 	*AR1W+CCGGI$Dax!1Iq	DMq		* GGBHHV$4tzz$ADKK$ORVRZRZ$Z[bbcfgjnjujuu
% ; :5s   8G>G! G&c                B    | j                  | j                  |            S )z5Return transformed label information for given index.)rL   get_image_and_label)rM   indexs     rQ   __getitem__zBaseDataset.__getitem__{  s    t77>??r   c                T   t        | j                  |         }|j                  dd       | j                  |      \  |d<   |d<   |d<   |d   d   |d   d   z  |d   d   |d   d   z  f|d<   | j                  r| j
                  | j                  |      |d	<   | j                  |      S )
z
        Get and return label information from the dataset.

        Args:
            index (int): Index of the image to retrieve.

        Returns:
            (dict[str, Any]): Label dictionary with image and metadata.
        r   Nimg	ori_shaperesized_shaper   r   	ratio_pad
rect_shape)r   r1   r   r   r5   r   r   update_labels_info)rM   r   labels      rQ   r   zBaseDataset.get_image_and_label  s     U+,		'4 CG??SXCY@eeK(%*@/"1%k(:1(==/"1%k(:1(==
k 99"&"3"3DJJu4E"FE,&&u--r   c                ,    t        | j                        S )z5Return the length of the labels list for the dataset.)r3   r1   rM   s    rQ   __len__zBaseDataset.__len__  s    4;;r   c                    |S )zCustom your label format here. )rM   r   s     rQ   r   zBaseDataset.update_labels_info  s    r   c                    t         )a  
        Users can customize augmentations here.

        Examples:
            >>> if self.augment:
            ...     # Training transforms
            ...     return Compose([])
            >>> else:
            ...    # Val transforms
            ...    return Compose([])
        NotImplementedError)rM   r    s     rQ   rK   zBaseDataset.build_transforms  s
     "!r   c                    t         )a)  
        Users can customize their own format here.

        Examples:
            Ensure output is a dictionary with the following keys:
            >>> dict(
            ...     im_file=im_file,
            ...     shape=shape,  # format: (height, width)
            ...     cls=cls,
            ...     bboxes=bboxes,  # xywh
            ...     segments=segments,  # xy
            ...     keypoints=keypoints,  # xy
            ...     normalized=True,  # or False
            ...     bbox_format="xyxy",  # or xywh, ltwh
            ... )
        r   r   s    rQ   r0   zBaseDataset.get_labels  s
    " "!r   )r#   str | list[str]r$   r   rE   z
bool | strr%   boolr    dict[str, Any]r'   rC   r5   r   r6   r   r7   r   r8   floatr&   r   rN   list[int] | Noner(   r   r)   r   )r#   r   returnz	list[str])r   r   r   None)T)r~   r   r   r   r   z3tuple[np.ndarray, tuple[int, int], tuple[int, int]])r   r   )r~   r   r   r   )r   )r   r   r   r   )r   r   r   r   )r   r   )r   r   r   r   )N)r    zdict[str, Any] | None)r   zlist[dict[str, Any]])__name__
__module____qualname____doc__r   r"   r.   r2   r   rI   r   rJ   rF   r9   r   r   r   r   rK   r0   __classcell__)rP   s   @rQ   r   r      s   .f !) $(L9!L9 L9 	L9
 L9 L9 L9 L9 L9 L9 L9 L9 "L9 L9 L9\$L004:l P"H<2@.* ""r   r   )!
__future__r   rc   r   r[   r   copyr   multiprocessing.poolr   pathlibr   typingr   r*   numpyrw   torch.utils.datar   ultralytics.data.utilsr	   r
   r   r   ultralytics.utilsr   r   r   r   r   ultralytics.utils.patchesr   r   r   r   rQ   <module>r     sH    #   	   +   
  $ ] ] P P ,d"' d"r   