
    vKgw                     $   S SK r S SKJr  S SKJrJr  S SKJr  S SKJ	r	J
r
JrJr  S SKJrJrJrJr   " S S5      r " S	 S
5      r " S S5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      r " S S\5      rg)    N)BytesIOreadableseekable)IN_MEMORY_UPLOAD_TAG)CompleteMultipartUploadTaskCreateMultipartUploadTaskSubmissionTaskTask)ChunksizeAdjusterDeferredOpenFileget_callbacksget_filtered_dictc                   0    \ rS rSrSS jrS rS rS rSrg)	AggregatedProgressCallback    c                 *    Xl         X l        SU l        g)a  Aggregates progress updates for every provided progress callback

:type callbacks: A list of functions that accepts bytes_transferred
    as a single argument
:param callbacks: The callbacks to invoke when threshold is reached

:type threshold: int
:param threshold: The progress threshold in which to take the
    aggregated progress and invoke the progress callback with that
    aggregated progress total
r   N)
_callbacks
_threshold_bytes_seen)self	callbacks	thresholds      Q/var/www/highfloat_scraper/venv/lib/python3.13/site-packages/s3transfer/upload.py__init__#AggregatedProgressCallback.__init__!   s     $#    c                     U =R                   U-  sl         U R                   U R                  :  a  U R                  5         g g N)r   r   _trigger_callbacks)r   bytes_transferreds     r   __call__#AggregatedProgressCallback.__call__1   s7    --t.##% /r   c                 F    U R                   S:  a  U R                  5         gg)z@Flushes out any progress that has not been sent to its callbacksr   N)r   r    r   s    r   flush AggregatedProgressCallback.flush6   s!    a##%  r   c                 X    U R                    H  nU" U R                  S9  M     SU l        g )N)r!   r   )r   r   )r   callbacks     r   r    -AggregatedProgressCallback._trigger_callbacks;   s&    Ht'7'78 (r   )r   r   r   N)i   )	__name__
__module____qualname____firstlineno__r   r"   r&   r    __static_attributes__ r   r   r   r       s     &
&
r   r   c                   J    \ rS rSrSrS rSS jrSS jrS rS r	S	 r
S
 rSrg)InterruptReaderA   a  Wrapper that can interrupt reading using an error

It uses a transfer coordinator to propagate an error if it notices
that a read is being made while the file is being read from.

:type fileobj: file-like obj
:param fileobj: The file-like object to read from

:type transfer_coordinator: s3transfer.futures.TransferCoordinator
:param transfer_coordinator: The transfer coordinator to use if the
    reader needs to be interrupted.
c                     Xl         X l        g r   _fileobj_transfer_coordinator)r   fileobjtransfer_coordinators      r   r   InterruptReader.__init__O   s    %9"r   Nc                     U R                   R                  (       a  U R                   R                  eU R                  R                  U5      $ r   )r7   	exceptionr6   read)r   amounts     r   r=   InterruptReader.readS   s:     %%//,,666}}!!&))r   c                 :    U R                   R                  X5        g r   )r6   seek)r   wherewhences      r   rA   InterruptReader.seek]   s    5)r   c                 6    U R                   R                  5       $ r   )r6   tellr%   s    r   rF   InterruptReader.tell`   s    }}!!##r   c                 8    U R                   R                  5         g r   )r6   closer%   s    r   rI   InterruptReader.closec   s    r   c                     U $ r   r0   r%   s    r   	__enter__InterruptReader.__enter__f   s    r   c                 $    U R                  5         g r   )rI   )r   argskwargss      r   __exit__InterruptReader.__exit__i   s    

r   r5   r   )r   )r+   r,   r-   r.   __doc__r   r=   rA   rF   rI   rL   rQ   r/   r0   r   r   r2   r2   A   s*    :**$r   r2   c                   b    \ rS rSrSrSS jr\S 5       rS rS r	S r
S	 rS
 rS rS rS rSrg)UploadInputManagerm   a"  Base manager class for handling various types of files for uploads

This class is typically used for the UploadSubmissionTask class to help
determine the following:

    * How to determine the size of the file
    * How to determine if a multipart upload is required
    * How to retrieve the body for a PutObject
    * How to retrieve the bodies for a set of UploadParts

The answers/implementations differ for the various types of file inputs
that may be accepted. All implementations must subclass and override
public methods from this class.
Nc                 (    Xl         X l        X0l        g r   )_osutilr7   _bandwidth_limiter)r   osutilr9   bandwidth_limiters       r   r   UploadInputManager.__init__}   s    %9""3r   c                     [        S5      e)zDetermines if the source for the upload is compatible with manager

:param upload_source: The source for which the upload will pull data
    from.

:returns: True if the manager can handle the type of source specified
    otherwise returns False.
zmust implement _is_compatible()NotImplementedErrorclsupload_sources     r   is_compatible UploadInputManager.is_compatible   s     ""CDDr   c                     [        S5      e)a  Whether the body it provides are stored in-memory

:type operation_name: str
:param operation_name: The name of the client operation that the body
    is being used for. Valid operation_names are ``put_object`` and
    ``upload_part``.

:rtype: boolean
:returns: True if the body returned by the manager will be stored in
    memory. False if the manager will not directly store the body in
    memory.
z%must implement store_body_in_memory()r^   r   operation_names     r   stores_body_in_memory(UploadInputManager.stores_body_in_memory   s     ""IJJr   c                     [        S5      e)zProvides the transfer size of an upload

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request
z&must implement provide_transfer_size()r^   r   transfer_futures     r   provide_transfer_size(UploadInputManager.provide_transfer_size   s     ""JKKr   c                     [        S5      e)a  Determines where a multipart upload is required

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request

:type config: s3transfer.manager.TransferConfig
:param config: The config associated to the transfer manager

:rtype: boolean
:returns: True, if the upload should be multipart based on
    configuration and size. False, otherwise.
z*must implement requires_multipart_upload()r^   r   rl   configs      r   requires_multipart_upload,UploadInputManager.requires_multipart_upload   s     ""NOOr   c                     [        S5      e)a  Returns the body to use for PutObject

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request

:type config: s3transfer.manager.TransferConfig
:param config: The config associated to the transfer manager

:rtype: s3transfer.utils.ReadFileChunk
:returns: A ReadFileChunk including all progress callbacks
    associated with the transfer future.
z$must implement get_put_object_body()r^   rk   s     r   get_put_object_body&UploadInputManager.get_put_object_body   s     ""HIIr   c                     [        S5      e)a  Yields the part number and body to use for each UploadPart

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The future associated with upload request

:type chunksize: int
:param chunksize: The chunksize to use for this upload.

:rtype: int, s3transfer.utils.ReadFileChunk
:returns: Yields the part number and the ReadFileChunk including all
    progress callbacks associated with the transfer future for that
    specific yielded part.
z)must implement yield_upload_part_bodies()r^   )r   rl   	chunksizes      r   yield_upload_part_bodies+UploadInputManager.yield_upload_part_bodies   s     ""MNNr   c                     [        XR                  5      nU R                  (       a$  U R                  R                  XR                  SS9nU$ )NF)enabled)r2   r7   rY   get_bandwith_limited_stream)r   r8   s     r   _wrap_fileobj UploadInputManager._wrap_fileobj   sI    !'+E+EF""--II33U J G r   c                 D    [        US5      nU(       a  [        U5      /$ / $ )Nprogress)r   r   )r   rl   r   s      r   _get_progress_callbacks*UploadInputManager._get_progress_callbacks   s&    !/:>	 .y9::	r   c                 B    U Vs/ sH  o"R                   PM     sn$ s  snf r   )r&   )r   aggregated_progress_callbacksr)   s      r   _get_close_callbacks'UploadInputManager._get_close_callbacks   s    /LM/L8/LMMMs   )rY   rX   r7   r   )r+   r,   r-   r.   rS   r   classmethodrc   rh   rm   rr   ru   ry   r~   r   r   r/   r0   r   r   rU   rU   m   sP    4
 	E 	EKLPJO Nr   rU   c                   ^    \ rS rSrSr\S 5       rS rS rS r	S r
S rS	 rS
 rS rS rSrg)UploadFilenameInputManager   zUpload utility for filenamesc                 "    [        U[        5      $ r   )
isinstancestrr`   s     r   rc   (UploadFilenameInputManager.is_compatible   s    ---r   c                     g)NFr0   rf   s     r   rh   0UploadFilenameInputManager.stores_body_in_memory   s    r   c                     UR                   R                  U R                  R                  UR                   R                  R
                  5      5        g r   )metarm   rX   get_file_size	call_argsr8   rk   s     r   rm   0UploadFilenameInputManager.provide_transfer_size   s:    22LL&&';';'E'E'M'MN	
r   c                 H    UR                   R                  UR                  :  $ r   )r   sizemultipart_thresholdrp   s      r   rr   4UploadFilenameInputManager.requires_multipart_upload   s    ##((F,F,FFFr   c                     U R                  U5      u  p#U R                  U5      nU R                  U5      nU R                  U5      nUR                  R
                  nU R                  R                  UUUUUS9$ )Nr8   
chunk_sizefull_file_sizer   close_callbacks)&_get_put_object_fileobj_with_full_sizer~   r   r   r   r   rX   #open_file_chunk_reader_from_fileobj)r   rl   r8   	full_sizer   r   r   s          r   ru   .UploadFilenameInputManager.get_put_object_body   s    !HH
 $$W-00A	33I>##(( ||??$+ @ 
 	
r   c           	   #     #    UR                   R                  nU R                  X5      n[        SUS-   5       H  nU R	                  U5      nU R                  U5      nX%S-
  -  nU R                  UR                   R                  R                  UUUS9u  pU R                  U	5      n	U R                  R                  U	UU
UUS9nX[4v   M     g 7f)N   )
start_byte	part_sizer   r   )r   r   _get_num_partsranger   r   '_get_upload_part_fileobj_with_full_sizer   r8   r~   rX   r   )r   rl   rx   r   	num_partspart_numberr   r   r   r8   r   read_file_chunks               r   ry   3UploadFilenameInputManager.yield_upload_part_bodies  s     (--22''C	 IM2K44_EI"77	BO"Ao6J "&!M!M$$..66%#-	 "N "G ((1G #llNN$(# / O O ..5 3s   CCc                 B    [        XU R                  R                  S9nU$ )N)open_function)r   rX   open)r   r8   r   s      r   _get_deferred_open_file2UploadFilenameInputManager._get_deferred_open_file1  s"    "t||/@/@
 r   c                     UR                   R                  R                  nUR                   R                  nU R	                  US5      U4$ )Nr   )r   r   r8   r   r   r   rl   r8   r   s       r   r   AUploadFilenameInputManager._get_put_object_fileobj_with_full_size7  sB    !&&0088##((++GQ7==r   c                 <    US   nUS   nU R                  X5      U4$ )Nr   r   )r   )r   r8   rP   r   r   s        r   r   BUploadFilenameInputManager._get_upload_part_fileobj_with_full_size<  s-    L)
+,	++G@)KKr   c                     [        [        R                  " UR                  R                  [        U5      -  5      5      $ r   )intmathceilr   r   float)r   rl   r   s      r   r   )UploadFilenameInputManager._get_num_partsA  s,    499_1166y9IIJKKr   r0   N)r+   r,   r-   r.   rS   r   rc   rh   rm   rr   ru   ry   r   r   r   r   r/   r0   r   r   r   r      sG    &. .

G
0/>>
L
Lr   r   c                   @    \ rS rSrSr\S 5       rS rS rS r	S r
Srg	)
UploadSeekableInputManageriE  z&Upload utility for an open file objectc                 <    [        U5      =(       a    [        U5      $ r   r   r`   s     r   rc   (UploadSeekableInputManager.is_compatibleH  s    &B8M+BBr   c                     US:X  a  gg)N
put_objectFTr0   rf   s     r   rh   0UploadSeekableInputManager.stores_body_in_memoryL  s    \)r   c                    UR                   R                  R                  nUR                  5       nUR	                  SS5        UR                  5       nUR	                  U5        UR                   R                  XC-
  5        g )Nr      )r   r   r8   rF   rA   rm   )r   rl   r8   start_positionend_positions        r   rm   0UploadSeekableInputManager.provide_transfer_sizeR  se    !&&0088 !Q||~^$22)	
r   c                 V    UR                  US   5      n[        U5      [        U5      4$ )Nr   )r=   r   len)r   r8   rP   datas       r   r   BUploadSeekableInputManager._get_upload_part_fileobj_with_full_size_  s,     ||F;/0 t}c$i''r   c                     UR                   R                  R                  nUR                  5       UR                   R                  -   nX#4$ r   )r   r   r8   rF   r   r   s       r   r   AUploadSeekableInputManager._get_put_object_fileobj_with_full_sizen  s?    !&&0088 ||~ 4 4 9 99}r   r0   N)r+   r,   r-   r.   rS   r   rc   rh   rm   r   r   r/   r0   r   r   r   r   E  s-    0C C
(r   r   c                   n   ^  \ rS rSrSrSU 4S jjr\S 5       rS rS r	S r
S rS	 rSS
 jrS rSrU =r$ )UploadNonSeekableInputManageriv  z7Upload utility for a file-like object that cannot seek.c                 4   > [         TU ]  XU5        SU l        g )Nr   )superr   _initial_data)r   rZ   r9   r[   	__class__s       r   r   &UploadNonSeekableInputManager.__init__y  s    7HI r   c                     [        U5      $ r   )r   r`   s     r   rc   +UploadNonSeekableInputManager.is_compatible}  s    &&r   c                     g)NTr0   rf   s     r   rh   3UploadNonSeekableInputManager.stores_body_in_memory  s    r   c                     g r   r0   rk   s     r   rm   3UploadNonSeekableInputManager.provide_transfer_size  s     	r   c                 2   UR                   R                  b#  UR                   R                  UR                  :  $ UR                   R                  R                  nUR                  nU R                  X4S5      U l        [        U R                  5      U:  a  gg)NFT)r   r   r   r   r8   _readr   r   )r   rl   rq   r8   r   s        r   rr   7UploadNonSeekableInputManager.requires_multipart_upload  s    $$0"'',,0J0JJJ "&&0088..	!ZZEBt!!"Y.r   c                     U R                  U5      nU R                  U5      nUR                  R                  R                  nU R                  U R                  UR                  5       -   X#5      nS U l        U$ r   )r   r   r   r   r8   
_wrap_datar   r=   )r   rl   r   r   r8   bodys         r   ru   1UploadNonSeekableInputManager.get_put_object_body  sl    00A	33I>!&&0088/
 "r   c              #     #    UR                   R                  R                  nSn U R                  U5      nU R	                  U5      nUS-  nU R                  X25      nU(       d  g U R                  XuU5      nS nXH4v   M[  7f)Nr   r   )r   r   r8   r   r   r   r   )	r   rl   rx   file_objectr   r   r   part_contentpart_objects	            r   ry   6UploadNonSeekableInputManager.yield_upload_part_bodies  s     %**44<< 44_EI"77	BO1K::k=L//K
  L** s   A?Bc                 l   [        U R                  5      S:X  a  UR                  U5      $ U[        U R                  5      ::  a,  U R                  SU nU(       a  U R                  US U l        U$ U[        U R                  5      -
  nU R                  UR                  U5      -   nU(       a  SU l        U$ )a  
Reads a specific amount of data from a stream and returns it. If there
is any data in initial_data, that will be popped out first.

:type fileobj: A file-like object that implements read
:param fileobj: The stream to read from.

:type amount: int
:param amount: The number of bytes to read from the stream.

:type truncate: bool
:param truncate: Whether or not to truncate initial_data after
    reading from it.

:return: Generator which generates part bodies from the initial data.
r   Nr   )r   r   r=   )r   r8   r>   truncater   amount_to_reads         r   r   #UploadNonSeekableInputManager._read  s    $ t!!"a'<<'' S++,,%%gv.D %)%7%7%@"K
  #d&8&8"99!!GLL$@@ !$Dr   c                     U R                  [        U5      5      nU R                  R                  U[	        U5      [	        U5      UUS9$ )ah  
Wraps data with the interrupt reader and the file chunk reader.

:type data: bytes
:param data: The data to wrap.

:type callbacks: list
:param callbacks: The callbacks associated with the transfer future.

:type close_callbacks: list
:param close_callbacks: The callbacks to be called when closing the
    wrapper for the data.

:return: Fully wrapped data.
r   )r~   r   rX   r   r   )r   r   r   r   r8   s        r   r   (UploadNonSeekableInputManager._wrap_data  sL      $$WT]3||??4yt9+ @ 
 	
r   )r   r   )T)r+   r,   r-   r.   rS   r   r   rc   rh   rm   rr   ru   ry   r   r   r/   __classcell__)r   s   @r   r   r   v  sH    A! ' '
"+((T
 
r   r   c                   X    \ rS rSrSr/ SQr/ SQrS r SS jrS r	S	 r
S
 rS rS rSrg)UploadSubmissionTaski   z.Task for submitting tasks to execute an upload)ChecksumAlgorithmSSECustomerKeySSECustomerAlgorithmSSECustomerKeyMD5RequestPayerExpectedBucketOwner)r   r   r   r   r   c                     [         [        [        /nUR                  R                  R
                  nU H  nUR                  U5      (       d  M  Us  $    [        SU S[        U5       S35      e)a?  Retrieves a class for managing input for an upload based on file type

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The transfer future for the request

:rtype: class of UploadInputManager
:returns: The appropriate class to use for managing a specific type of
    input for uploads.
zInput z
 of type: z is not supported.)	r   r   r   r   r   r8   rc   RuntimeErrortype)r   rl   upload_manager_resolver_chainr8   upload_manager_clss        r   _get_upload_input_manager_cls2UploadSubmissionTask._get_upload_input_manager_cls  sw     '&))
% "&&0088"?!//88)) #@ WIZW6HI
 	
r   Nc                    U R                  U5      " X0R                  U5      nUR                  R                  c  UR	                  U5        UR                  XR5      (       d  U R                  UUUUUU5        gU R                  UUUUUU5        g)aj  
:param client: The client associated with the transfer manager

:type config: s3transfer.manager.TransferConfig
:param config: The transfer config associated with the transfer
    manager

:type osutil: s3transfer.utils.OSUtil
:param osutil: The os utility associated to the transfer manager

:type request_executor: s3transfer.futures.BoundedExecutor
:param request_executor: The request executor associated with the
    transfer manager

:type transfer_future: s3transfer.futures.TransferFuture
:param transfer_future: The transfer future associated with the
    transfer request that tasks are being submitted for
N)r   r7   r   r   rm   rr   _submit_upload_request_submit_multipart_request)r   clientrq   rZ   request_executorrl   r[   upload_input_managers           r   _submitUploadSubmissionTask._submit,  s    6  $AA 

,,.? A
 $$, 66G $==
 
 '' $ ** $r   c                    UR                   R                  nU R                  US5      nU R                  R	                  U[        U R                  UUR                  U5      UR                  UR                  UR                  S.SS9US9  g )Nr   )r  r8   bucketkey
extra_argsT)r9   main_kwargsis_finaltag)
r   r   _get_upload_task_tagr7   submitPutObjectTaskru   r	  r
  r  )	r   r  rq   rZ   r  rl   r  r   put_object_tags	            r   r  +UploadSubmissionTask._submit_upload_requeste  s     $((22	 22 ,

 	""))%)%?%?$3GG'  (..$=="+"6"6   	* 	
r   c                 d   UR                   R                  nU R                  R                  U[	        U R                  UUR
                  UR                  UR                  S.S95      n/ n	U R                  UR                  5      n
U R                  US5      nUR                   R                  n[        5       nUR                  UR                  U5      nUR                  X^5      nU H`  u  nnU	R                  U R                  R                  U[!        U R                  UUUR
                  UR                  UU
S.SU0S9US95        Mb     U R#                  UR                  5      nU R                  R                  U[%        U R                  UUR
                  UR                  US.UU	S.S	S
95        g )N)r  r	  r
  r  )r9   r  upload_part)r  r8   r	  r
  r   r  	upload_id)r9   r  pending_main_kwargsr  )r  partsT)r9   r  r  r  )r   r   r7   r  r	   r	  r
  r  _extra_upload_part_argsr  r   r   adjust_chunksizemultipart_chunksizery   appendUploadPartTask_extra_complete_multipart_argsr   )r   r  rq   rZ   r  rl   r  r   create_multipart_futurepart_futuresextra_part_argsupload_part_tagr   adjusterrx   part_iteratorr   r8   complete_multipart_extra_argss                      r   r  .UploadSubmissionTask._submit_multipart_request  s    $((22	 #'"<"<"C"C%%)%?%?$'..$=="+"6"6	#
 66y7K7KL 33 -
 ##(($&--f.H.H$O	,EE
 %2 K**11$"-1-G-G&,'.&/&6&6#,==+6*9% ()@- (! 2  %2, )-(K(K  )
% 	""))'%)%?%?$'..$=="?	 "9)% 	
r   c                 ,    [        XR                  5      $ r   )r   UPLOAD_PART_ARGSr   r  s     r   r  ,UploadSubmissionTask._extra_upload_part_args  s     !-B-BCCr   c                 ,    [        XR                  5      $ r   )r   COMPLETE_MULTIPART_ARGSr*  s     r   r  3UploadSubmissionTask._extra_complete_multipart_args  s     -I-IJJr   c                 B    S nUR                  U5      (       a  [        nU$ r   )rh   r   )r   r  rg   r  s       r   r  )UploadSubmissionTask._get_upload_task_tag  s"    55nEE&C
r   r0   r   )r+   r,   r-   r.   rS   r)  r-  r   r  r  r  r  r  r  r/   r0   r   r   r   r      sC    8
> 7r!
FT
lD
Kr   r   c                       \ rS rSrSrS rSrg)r  i  z Task to do a nonmultipart uploadc                 f    U nUR                   " SX4US.UD6  SSS5        g! , (       d  f       g= f)a  
:param client: The client to use when calling PutObject
:param fileobj: The file to upload.
:param bucket: The name of the bucket to upload to
:param key: The name of the key to upload to
:param extra_args: A dictionary of any extra arguments that may be
    used in the upload.
)BucketKeyBodyNr0   )r   )r   r  r8   r	  r
  r  r   s          r   _mainPutObjectTask._main  s+     NV4N:N WWs   "
0r0   Nr+   r,   r-   r.   rS   r6  r/   r0   r   r   r  r    s    *
Or   r  c                       \ rS rSrSrS rSrg)r  i  z+Task to upload a part in a multipart uploadc           
          U nUR                   " SUUUUUS.UD6n	SSS5        W	S   n
XS.nSU;   a#  US   R                  5       nSU 3nX;   a  X   X'   U$ ! , (       d  f       NB= f)a[  
:param client: The client to use when calling PutObject
:param fileobj: The file to upload.
:param bucket: The name of the bucket to upload to
:param key: The name of the key to upload to
:param upload_id: The id of the upload
:param part_number: The number representing the part of the multipart
    upload
:param extra_args: A dictionary of any extra arguments that may be
    used in the upload.

:rtype: dict
:returns: A dictionary representing a part::

    {'Etag': etag_value, 'PartNumber': part_number}

    This value can be appended to a list to be used to complete
    the multipart upload.
)r3  r4  UploadId
PartNumberr5  NETag)r=  r<  r   Checksumr0   )r  upper)r   r  r8   r	  r
  r  r   r  r   responseetagpart_metadataalgorithm_namechecksum_members                 r   r6  UploadPartTask._main   s    , )) "& H  !%A*,'(;<BBDN ((89O*191J.! Ws   A
A&r0   Nr8  r0   r   r   r  r    s
    5&r   r  )r   ior   s3transfer.compatr   r   s3transfer.futuresr   s3transfer.tasksr   r	   r
   r   s3transfer.utilsr   r   r   r   r   r2   rU   r   r   r   r   r  r  r0   r   r   <module>rK     s      0 3   B) )XxN xNvZL!3 ZLz.!; .bG
$6 G
Tj> jZOD O )T )r   