.h;\ddlZddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl m Z edZe dvr"ejrej!d Gd d ZGd d Zedk(rededeZ ej-Zeddededddededddededdddededd d!ded"edd#dd$ed%ed&d'd(ded)ed&d*d(ded+rLed,ed+j1D]-\ZZed-ed.ed/d0d1ed2dd3ed4d5d6ed7d0d8 /ned9ej6d:y#e$r ed;YywxYw)<N)datetime)Path)MACOSRANK)check_requirementsz train.log>rT) missing_okcxeZdZdZdZdZdZdZdZdZ dZ Gd d Z Gd d e jZy ) ConsoleLoggera Console output capture with API/file streaming and deduplication. Captures stdout/stderr output and streams it to either an API endpoint or local file, with intelligent deduplication to reduce noise from repetitive console output. Attributes: destination (str | Path): Target destination for streaming (URL or Path object). is_api (bool): Whether destination is an API endpoint (True) or local file (False). original_stdout: Reference to original sys.stdout for restoration. original_stderr: Reference to original sys.stderr for restoration. log_queue (queue.Queue): Thread-safe queue for buffering log messages. active (bool): Whether console capture is currently active. worker_thread (threading.Thread): Background thread for processing log queue. last_line (str): Last processed line for deduplication. last_time (float): Timestamp of last processed line. last_progress_line (str): Last progress bar line for progress deduplication. last_was_progress (bool): Whether the last line was a progress bar. Examples: Basic file logging: >>> logger = ConsoleLogger("training.log") >>> logger.start_capture() >>> print("This will be logged") >>> logger.stop_capture() API streaming: >>> logger = ConsoleLogger("https://api.example.com/logs") >>> logger.start_capture() >>> # All output streams to API >>> logger.stop_capture() cx||_t|txr|jd|_|jst ||_t j|_t j|_ tjd|_ d|_d|_d|_d|_d|_d|_y)z Initialize with API endpoint or local file path. Args: destination (str | Path): API endpoint URL (http/https) or local file path for streaming output. )zhttp://zhttps://)maxsizeFNg) destination isinstancestr startswithis_apirsysstdoutoriginal_stdoutstderroriginal_stderrqueueQueue log_queueactive worker_thread last_line last_timelast_progress_linelast_was_progress)selfrs V/var/www/html/ai-service/venv/lib/python3.12/site-packages/ultralytics/utils/logger.py__init__zConsoleLogger.__init__7s' c2f{7M7MNe7f {{#K0D  #zz"zzT2 !"$!&c|jryd|_|j|j|jt_|j|j |jt_ |j|j}tjdj|tj|jd|_|j j#y#t$rYLwxYw)zTStart capturing console output and redirect stdout/stderr to custom capture objects.NT ultralytics)targetdaemon)r_ConsoleCapturer _queue_logrrrr _LogHandlerlogging getLogger addHandler Exception threadingThread_stream_workerrstart)r#handlers r$ start_capturezConsoleLogger.start_capturePs ;;  ))$*>*>P ))$*>*>P  &&t7G   m , 7 7 @'--T5H5HQUV   "    s4?C44 D?Dc|jsyd|_|jt_|jt_|j jdy)zAStop capturing console output and restore original stdout/stderr.NF)rrrrrrrputr#s r$ stop_capturezConsoleLogger.stop_capturecs?{{  )) ))  4 r&c|jsytj}d|vr|jdd}|jd}|r|ddk(r|j|D]}|j }d|vrd|vrM|jddj }||j k(r |jr[||_d |_n|s|jrd |_d |_||jk(r||jz d kr||_||_ |jd s+tjjd }d|d|}|j|dr  y)z?Queue console text with deduplication and timestamp processing.N r ru─u ━━rTFg?z[20z%Y-%m-%d %H:%M:%S[z] )rtimesplitpoprstripstripr!r"rr rrnowstrftime _safe_put)r#text current_timelinesline progress_core timestamps r$r,zConsoleLogger._queue_logmsj{{ yy{  4<::d#B'D 4  U2Y"_ IIK# D;;=D}D $ 9 5a 8 > > @  D$;$;;@V@V*7')-& 6 6-2D*).&t~~%,*G#*M!DN)DN??5)$LLN334GH 9+Rv.>>TF"+.G# r&c |jj|y#tj$rS |jj |jj|Yy#tj $rYYywxYwwxYw)z0Safely put item in queue with overflow handling.TF)r put_nowaitrFull get_nowaitEmpty)r#items r$rGzConsoleLogger._safe_putso  NN % %d +zz  ))+))$/;;    s&B5A))B<B?BBc|jr? |jjd}|y|j||jr>yy#tj $rYawxYw)z4Background worker for streaming logs to destination.)timeoutN)rrget _write_logrrR)r#log_texts r$r4zConsoleLogger._stream_workers[kk >>--a-8#) kk ;;  sA A A#"A#c |jr_ddl}tjj |j d}|j t|j|dy|jjjdd|jjdd 5}|j|dddy#1swYyxYw#t$r$}td ||j Yd}~yd}~wwxYw) z4Write log to API endpoint or local file destination.rN)rMmessage)jsonrVT)parentsexist_okazutf-8)encodingzPlatform logging error: )file)rrequestsrrE isoformatrDpostrrparentmkdiropenwriter1printr)r#rHrcpayloadfes r$rXzConsoleLogger._write_logs M{{(0 (@(@(Btzz|\ c$"2"23'1 M  ''--dT-J%%**3*A"QGGDM""" M ,QC0t7K7K L L Ms=A*C-AC1C C CCC D!DDc&eZdZdZdZdZdZdZy)ConsoleLogger._ConsoleCapturez"Lightweight stdout/stderr capture.originalcallbackc ||_||_yNrp)r#rqrrs r$r%z&ConsoleLogger._ConsoleCapture.__init__s$DM$DMr&c\|jj||j|yrt)rqrirr)r#rHs r$riz#ConsoleLogger._ConsoleCapture.writes MM   % MM$ r&c8|jjyrt)rqflushr:s r$rwz#ConsoleLogger._ConsoleCapture.flushs MM   !r&N)__name__ __module__ __qualname____doc__ __slots__r%rirwr&r$r+ros0,  %  "r&r+c,eZdZdZdZfdZdZxZS)ConsoleLogger._LogHandlerzLightweight logging handler.)rrc0t|||_yrt)superr%rr)r#rr __class__s r$r%z"ConsoleLogger._LogHandler.__init__s G  $DMr&cJ|j|j|dzy)Nr>)rrformat)r#records r$emitzConsoleLogger._LogHandler.emits MM$++f-4 5r&)rxryrzr{r|r%r __classcell__)rs@r$r-rs*!  % 6r&r-N)rxryrzr{r%r7r;r,rGr4rXr+r.Handlerr-r}r&r$r r sGB'2#&!2h   M"" 6goo 6r&r c(eZdZdZdZdZdZdZy) SystemLoggeru Log dynamic system metrics for training monitoring. Captures real-time system metrics including CPU, RAM, disk I/O, network I/O, and NVIDIA GPU statistics for training performance monitoring and analysis. Attributes: pynvml: NVIDIA pynvml module instance if successfully imported, None otherwise. nvidia_initialized (bool): Whether NVIDIA GPU monitoring is available and initialized. net_start: Initial network I/O counters for calculating cumulative usage. disk_start: Initial disk I/O counters for calculating cumulative usage. Examples: Basic usage: >>> logger = SystemLogger() >>> metrics = logger.get_metrics() >>> print(f"CPU: {metrics['cpu']}%, RAM: {metrics['ram']}%") >>> if metrics["gpus"]: ... gpu0 = metrics["gpus"]["0"] ... print(f"GPU0: {gpu0['usage']}% usage, {gpu0['temp']}°C") Training loop integration: >>> system_logger = SystemLogger() >>> for epoch in range(epochs): ... # Training code here ... metrics = system_logger.get_metrics() ... # Log to database/file cddl}d|_|j|_|j |_|j |_y)zInitialize the system logger.rN)psutilpynvml _init_nvidianvidia_initializednet_io_counters net_startdisk_io_counters disk_start)r#rs r$r%zSystemLogger.__init__s= "&"3"3"5//1 113r&c trJtdtd|_|jj y#t $rYywxYw)z-Initialize NVIDIA GPU monitoring with pynvml.znvidia-ml-py>=12.0.0rTF)rr __import__rnvmlInitr1r:s r$rzSystemLogger._init_nvidia sI  9 5 6$X.DK KK "  s=A A  A cddl}|j}|j}|j}t j d}t |jdt |jdt |j|jjz dz dt |j|jjz dz dt |jdz ddt |j|jjz dz dt |j|jjz dz ddid }|j r"|d j#|j%|S) a Get current system metrics. Collects comprehensive system metrics including CPU usage, RAM usage, disk I/O statistics, network I/O statistics, and GPU metrics (if available). Example output: ```python metrics = { "cpu": 45.2, "ram": 78.9, "disk": {"read_mb": 156.7, "write_mb": 89.3, "used_gb": 256.8}, "network": {"recv_mb": 157.2, "sent_mb": 89.1}, "gpus": { 0: {"usage": 95.6, "memory": 85.4, "temp": 72, "power": 285}, 1: {"usage": 94.1, "memory": 82.7, "temp": 70, "power": 278}, }, } ``` - cpu (float): CPU usage percentage (0-100%) - ram (float): RAM usage percentage (0-100%) - disk (dict): - read_mb (float): Cumulative disk read in MB since initialization - write_mb (float): Cumulative disk write in MB since initialization - used_gb (float): Total disk space used in GB - network (dict): - recv_mb (float): Cumulative network received in MB since initialization - sent_mb (float): Cumulative network sent in MB since initialization - gpus (dict): GPU metrics by device index (e.g., 0, 1) containing: - usage (int): GPU utilization percentage (0-100%) - memory (float): CUDA memory usage percentage (0-100%) - temp (int): GPU temperature in degrees Celsius - power (int): GPU power consumption in watts Returns: metrics (dict): System metrics containing 'cpu', 'ram', 'disk', 'network', 'gpus' with respective usage data. rN/ii@)read_mbwrite_mbused_gb)recv_mbsent_mb)cpuramdisknetworkgpusr)rrrvirtual_memoryshutil disk_usageround cpu_percentpercent read_bytesr write_bytesused bytes_recvr bytes_sentrupdate_get_nvidia_metrics)r#rnetrmemoryrmetricss r$ get_metricszSystemLogger.get_metricssML $$&&&(&&(&&s+ ++-q1+ $//DOO4N4N"NSZ![]^_!4#3#3doo6Q6Q#QV]"^`ab G!>3L3L"LQX!Y[\] #..4>>3L3L"LQX!Y[\]    " " FO " "4#;#;#= >r&ci}|jr |js|S |jj}t|D]}|jj |}|jj |}|jj |}|jj||jj}|jj|dz}t|jdt|j|jz dzd||d|t|< |S#t$rY|SwxYw)zMGet NVIDIA GPU metrics including utilization, memory, temperature, and power.r rd)usagertemppower)rrnvmlDeviceGetCountrangenvmlDeviceGetHandleByIndexnvmlDeviceGetUtilizationRatesnvmlDeviceGetMemoryInfonvmlDeviceGetTemperatureNVML_TEMPERATURE_GPUnvmlDeviceGetPowerUsagergpurtotalrr1) r#r device_countihandleutilrrrs r$rz SystemLogger._get_nvidia_metricsYs&&dkkK ;;99;L<( ??B{{@@H<rsE  )7 $7?'..0t,M6M6`JJZ z 23 "# ^F )((*G .b ) E'%..a0 1 E'%..a0 1 K :4@D E L!