
    `2f)w                     $   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlmZmZmZmZ  G d d      Z G d	 d
      Z G d d      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Zy)    N)BytesIOreadableseekable)IN_MEMORY_UPLOAD_TAG)CompleteMultipartUploadTaskCreateMultipartUploadTaskSubmissionTaskTask)ChunksizeAdjusterDeferredOpenFileget_callbacksget_filtered_dictc                   &    e Zd ZddZd Zd Zd Zy)AggregatedProgressCallbackc                 .    || _         || _        d| _        y)a  Aggregates progress updates for every provided progress callback

        :type callbacks: A list of functions that accepts bytes_transferred
            as a single argument
        :param callbacks: The callbacks to invoke when threshold is reached

        :type threshold: int
        :param threshold: The progress threshold in which to take the
            aggregated progress and invoke the progress callback with that
            aggregated progress total
        r   N)
_callbacks
_threshold_bytes_seen)self	callbacks	thresholds      Q/var/www/highfloat_scraper/venv/lib/python3.12/site-packages/s3transfer/upload.py__init__z#AggregatedProgressCallback.__init__!   s     $#    c                     | xj                   |z  c_         | j                   | j                  k\  r| j                          y y N)r   r   _trigger_callbacks)r   bytes_transferreds     r   __call__z#AggregatedProgressCallback.__call__1   s7    --t.##% /r   c                 D    | j                   dkD  r| j                          yy)z@Flushes out any progress that has not been sent to its callbacksr   N)r   r   r   s    r   flushz AggregatedProgressCallback.flush6   s!    a##%  r   c                 Z    | j                   D ]  } || j                          d| _        y )N)r   r   )r   r   )r   callbacks     r   r   z-AggregatedProgressCallback._trigger_callbacks;   s+     	9Ht'7'78	9r   N)i   )__name__
__module____qualname__r   r    r#   r    r   r   r   r       s     &
&
r   r   c                   >    e Zd ZdZd Zd
dZddZd Zd Zd Z	d	 Z
y)InterruptReadera  Wrapper that can interrupt reading using an error

    It uses a transfer coordinator to propagate an error if it notices
    that a read is being made while the file is being read from.

    :type fileobj: file-like obj
    :param fileobj: The file-like object to read from

    :type transfer_coordinator: s3transfer.futures.TransferCoordinator
    :param transfer_coordinator: The transfer coordinator to use if the
        reader needs to be interrupted.
    c                      || _         || _        y r   )_fileobj_transfer_coordinator)r   fileobjtransfer_coordinators      r   r   zInterruptReader.__init__O   s    %9"r   Nc                     | j                   j                  r| j                   j                  | j                  j                  |      S r   )r.   	exceptionr-   read)r   amounts     r   r3   zInterruptReader.readS   s:     %%//,,666}}!!&))r   c                 <    | j                   j                  ||       y r   )r-   seek)r   wherewhences      r   r6   zInterruptReader.seek]   s    5&)r   c                 6    | j                   j                         S r   )r-   tellr"   s    r   r:   zInterruptReader.tell`   s    }}!!##r   c                 8    | j                   j                          y r   )r-   closer"   s    r   r<   zInterruptReader.closec   s    r   c                     | S r   r)   r"   s    r   	__enter__zInterruptReader.__enter__f   s    r   c                 $    | j                          y r   )r<   )r   argskwargss      r   __exit__zInterruptReader.__exit__i   s    

r   r   )r   )r&   r'   r(   __doc__r   r3   r6   r:   r<   r>   rB   r)   r   r   r+   r+   A   s*    :**$r   r+   c                   X    e Zd ZdZddZed        Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zy)UploadInputManageraJ  Base manager class for handling various types of files for uploads

    This class is typically used for the UploadSubmissionTask class to help
    determine the following:

        * How to determine the size of the file
        * How to determine if a multipart upload is required
        * How to retrieve the body for a PutObject
        * How to retrieve the bodies for a set of UploadParts

    The answers/implementations differ for the various types of file inputs
    that may be accepted. All implementations must subclass and override
    public methods from this class.
    Nc                 .    || _         || _        || _        y r   )_osutilr.   _bandwidth_limiter)r   osutilr0   bandwidth_limiters       r   r   zUploadInputManager.__init__}   s    %9""3r   c                     t        d      )a  Determines if the source for the upload is compatible with manager

        :param upload_source: The source for which the upload will pull data
            from.

        :returns: True if the manager can handle the type of source specified
            otherwise returns False.
        zmust implement _is_compatible()NotImplementedErrorclsupload_sources     r   is_compatiblez UploadInputManager.is_compatible   s     ""CDDr   c                     t        d      )a  Whether the body it provides are stored in-memory

        :type operation_name: str
        :param operation_name: The name of the client operation that the body
            is being used for. Valid operation_names are ``put_object`` and
            ``upload_part``.

        :rtype: boolean
        :returns: True if the body returned by the manager will be stored in
            memory. False if the manager will not directly store the body in
            memory.
        z%must implement store_body_in_memory()rL   r   operation_names     r   stores_body_in_memoryz(UploadInputManager.stores_body_in_memory   s     ""IJJr   c                     t        d      )zProvides the transfer size of an upload

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request
        z&must implement provide_transfer_size()rL   r   transfer_futures     r   provide_transfer_sizez(UploadInputManager.provide_transfer_size   s     ""JKKr   c                     t        d      )a  Determines where a multipart upload is required

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: boolean
        :returns: True, if the upload should be multipart based on
            configuration and size. False, otherwise.
        z*must implement requires_multipart_upload()rL   r   rX   configs      r   requires_multipart_uploadz,UploadInputManager.requires_multipart_upload   s     ""NOOr   c                     t        d      )a  Returns the body to use for PutObject

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: s3transfer.utils.ReadFileChunk
        :returns: A ReadFileChunk including all progress callbacks
            associated with the transfer future.
        z$must implement get_put_object_body()rL   rW   s     r   get_put_object_bodyz&UploadInputManager.get_put_object_body   s     ""HIIr   c                     t        d      )a  Yields the part number and body to use for each UploadPart

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type chunksize: int
        :param chunksize: The chunksize to use for this upload.

        :rtype: int, s3transfer.utils.ReadFileChunk
        :returns: Yields the part number and the ReadFileChunk including all
            progress callbacks associated with the transfer future for that
            specific yielded part.
        z)must implement yield_upload_part_bodies()rL   )r   rX   	chunksizes      r   yield_upload_part_bodiesz+UploadInputManager.yield_upload_part_bodies   s     ""MNNr   c                     t        || j                        }| j                  r(| j                  j                  || j                  d      }|S )NF)enabled)r+   r.   rH   get_bandwith_limited_stream)r   r/   s     r   _wrap_fileobjz UploadInputManager._wrap_fileobj   sM    !'4+E+EF""--II33U J G r   c                 :    t        |d      }|rt        |      gS g S )Nprogress)r   r   )r   rX   r   s      r   _get_progress_callbacksz*UploadInputManager._get_progress_callbacks   s&    !/:>	 .y9::	r   c                 @    |D cg c]  }|j                    c}S c c}w r   )r#   )r   aggregated_progress_callbacksr%   s      r   _get_close_callbacksz'UploadInputManager._get_close_callbacks   s    /LM8MMMs   r   )r&   r'   r(   rC   r   classmethodrQ   rU   rY   r]   r_   rb   rf   ri   rl   r)   r   r   rE   rE   m   sP    4
 	E 	EKLPJO Nr   rE   c                   V    e Zd ZdZed        Zd Zd Zd Zd Z	d Z
d Zd	 Zd
 Zd Zy)UploadFilenameInputManagerzUpload utility for filenamesc                 "    t        |t              S r   )
isinstancestrrN   s     r   rQ   z(UploadFilenameInputManager.is_compatible   s    ---r   c                      y)NFr)   rS   s     r   rU   z0UploadFilenameInputManager.stores_body_in_memory   s    r   c                     |j                   j                  | j                  j                  |j                   j                  j
                               y r   )metarY   rG   get_file_size	call_argsr/   rW   s     r   rY   z0UploadFilenameInputManager.provide_transfer_size   s:    22LL&&';';'E'E'M'MN	
r   c                 H    |j                   j                  |j                  k\  S r   )ru   sizemultipart_thresholdr[   s      r   r]   z4UploadFilenameInputManager.requires_multipart_upload   s    ##((F,F,FFFr   c                     | j                  |      \  }}| j                  |      }| j                  |      }| j                  |      }|j                  j
                  }| j                  j                  |||||      S )Nr/   
chunk_sizefull_file_sizer   close_callbacks)&_get_put_object_fileobj_with_full_sizerf   ri   rl   ru   ry   rG   #open_file_chunk_reader_from_fileobj)r   rX   r/   	full_sizer   r   ry   s          r   r_   z.UploadFilenameInputManager.get_put_object_body   s    !HH
 $$W-00A	33I>##(( ||??$+ @ 
 	
r   c              #     K   |j                   j                  }| j                  ||      }t        d|dz         D ]  }| j	                  |      }| j                  |      }||dz
  z  }| j                  |j                   j                  j                  |||      \  }	}
| j                  |	      }	| j                  j                  |	||
||      }||f  y w)N   )
start_byte	part_sizer~   r|   )ru   ry   _get_num_partsrangeri   rl   '_get_upload_part_fileobj_with_full_sizerw   r/   rf   rG   r   )r   rX   ra   r~   	num_partspart_numberr   r   r   r/   r   read_file_chunks               r   rb   z3UploadFilenameInputManager.yield_upload_part_bodies  s     (--22''C	 IM2 	/K44_EI"77	BO"kAo6J "&!M!M$$..66%#-	 "N "GY ((1G #llNN$(# / O O ..5	/s   CCc                 J    t        ||| j                  j                        }|S )N)open_function)r   rG   open)r   r/   r   s      r   _get_deferred_open_filez2UploadFilenameInputManager._get_deferred_open_file1  s$    "Zt||/@/@
 r   c                     |j                   j                  j                  }|j                   j                  }| j	                  |d      |fS )Nr   )ru   rw   r/   ry   r   r   rX   r/   ry   s       r   r   zAUploadFilenameInputManager._get_put_object_fileobj_with_full_size7  sB    !&&0088##((++GQ7==r   c                 >    |d   }|d   }| j                  ||      |fS )Nr   r~   )r   )r   r/   rA   r   r   s        r   r   zBUploadFilenameInputManager._get_upload_part_fileobj_with_full_size<  s/    L)
+,	++GZ@)KKr   c                 ~    t        t        j                  |j                  j                  t        |      z              S r   )intmathceilru   ry   float)r   rX   r   s      r   r   z)UploadFilenameInputManager._get_num_partsA  s,    499_1166y9IIJKKr   N)r&   r'   r(   rC   rm   rQ   rU   rY   r]   r_   rb   r   r   r   r   r)   r   r   ro   ro      sG    &. .

G
0/>>
L
Lr   ro   c                   8    e Zd ZdZed        Zd Zd Zd Zd Z	y)UploadSeekableInputManagerz&Upload utility for an open file objectc                 2    t        |      xr t        |      S r   r   rN   s     r   rQ   z(UploadSeekableInputManager.is_compatibleH  s    &B8M+BBr   c                     |dk(  ryy)N
put_objectFTr)   rS   s     r   rU   z0UploadSeekableInputManager.stores_body_in_memoryL  s    \)r   c                    |j                   j                  j                  }|j                         }|j	                  dd       |j                         }|j	                  |       |j                   j                  ||z
         y )Nr      )ru   rw   r/   r:   r6   rY   )r   rX   r/   start_positionend_positions        r   rY   z0UploadSeekableInputManager.provide_transfer_sizeR  sg    !&&0088 !Q||~^$22>)	
r   c                 V    |j                  |d         }t        |      t        |      fS )Nr   )r3   r   len)r   r/   rA   datas       r   r   zBUploadSeekableInputManager._get_upload_part_fileobj_with_full_size_  s,     ||F;/0 t}c$i''r   c                     |j                   j                  j                  }|j                         |j                   j                  z   }||fS r   )ru   rw   r/   r:   ry   r   s       r   r   zAUploadSeekableInputManager._get_put_object_fileobj_with_full_sizen  sA    !&&0088 ||~ 4 4 9 99}r   N)
r&   r'   r(   rC   rm   rQ   rU   rY   r   r   r)   r   r   r   r   E  s-    0C C
(r   r   c                   `     e Zd ZdZd fd	Zed        Zd Zd Zd Z	d Z
d Zdd	Zd
 Z xZS )UploadNonSeekableInputManagerz7Upload utility for a file-like object that cannot seek.c                 6    t         |   |||       d| _        y )Nr   )superr   _initial_data)r   rI   r0   rJ   	__class__s       r   r   z&UploadNonSeekableInputManager.__init__y  s    !57HI r   c                     t        |      S r   )r   rN   s     r   rQ   z+UploadNonSeekableInputManager.is_compatible}  s    &&r   c                      y)NTr)   rS   s     r   rU   z3UploadNonSeekableInputManager.stores_body_in_memory  s    r   c                      y r   r)   rW   s     r   rY   z3UploadNonSeekableInputManager.provide_transfer_size  s     	r   c                 0   |j                   j                  #|j                   j                  |j                  k\  S |j                   j                  j                  }|j                  }| j                  ||d      | _        t        | j                        |k  ryy)NFT)ru   ry   rz   rw   r/   _readr   r   )r   rX   r\   r/   r   s        r   r]   z7UploadNonSeekableInputManager.requires_multipart_upload  s    $$0"'',,0J0JJJ "&&0088..	!ZZEBt!!"Y.r   c                     | j                  |      }| j                  |      }|j                  j                  j                  }| j                  | j                  |j                         z   ||      }d | _        |S r   )ri   rl   ru   rw   r/   
_wrap_datar   r3   )r   rX   r   r   r/   bodys         r   r_   z1UploadNonSeekableInputManager.get_put_object_body  sn    00A	33I>!&&0088/O
 "r   c              #      K   |j                   j                  j                  }d}	 | j                  |      }| j	                  |      }|dz  }| j                  ||      }|sy | j                  |||      }d }||f Xw)Nr   r   )ru   rw   r/   ri   rl   r   r   )	r   rX   ra   file_objectr   r   r   part_contentpart_objects	            r   rb   z6UploadNonSeekableInputManager.yield_upload_part_bodies  s     %**44<< 44_EI"77	BO1K::k9=L//iK
  L{** s   A<A>c                 T   t        | j                        dk(  r|j                  |      S |t        | j                        k  r'| j                  d| }|r| j                  |d | _        |S |t        | j                        z
  }| j                  |j                  |      z   }|rd| _        |S )a=  
        Reads a specific amount of data from a stream and returns it. If there
        is any data in initial_data, that will be popped out first.

        :type fileobj: A file-like object that implements read
        :param fileobj: The stream to read from.

        :type amount: int
        :param amount: The number of bytes to read from the stream.

        :type truncate: bool
        :param truncate: Whether or not to truncate initial_data after
            reading from it.

        :return: Generator which generates part bodies from the initial data.
        r   Nr   )r   r   r3   )r   r/   r4   truncater   amount_to_reads         r   r   z#UploadNonSeekableInputManager._read  s    $ t!!"a'<<'' S++,,%%gv.D %)%7%7%@"K
  #d&8&8"99!!GLL$@@ !$Dr   c                     | j                  t        |            }| j                  j                  |t	        |      t	        |      ||      S )a  
        Wraps data with the interrupt reader and the file chunk reader.

        :type data: bytes
        :param data: The data to wrap.

        :type callbacks: list
        :param callbacks: The callbacks associated with the transfer future.

        :type close_callbacks: list
        :param close_callbacks: The callbacks to be called when closing the
            wrapper for the data.

        :return: Fully wrapped data.
        r|   )rf   r   rG   r   r   )r   r   r   r   r/   s        r   r   z(UploadNonSeekableInputManager._wrap_data  sL      $$WT]3||??4yt9+ @ 
 	
r   r   )T)r&   r'   r(   rC   r   rm   rQ   rU   rY   r]   r_   rb   r   r   __classcell__)r   s   @r   r   r   v  sC    A! ' '
"+((T
r   r   c                   N    e Zd ZdZg dZg dZd Z	 ddZd Zd Z	d	 Z
d
 Zd Zy)UploadSubmissionTaskz.Task for submitting tasks to execute an upload)ChecksumAlgorithmSSECustomerKeySSECustomerAlgorithmSSECustomerKeyMD5RequestPayerExpectedBucketOwner)r   r   r   r   r   c                     t         t        t        g}|j                  j                  j
                  }|D ]  }|j                  |      s|c S  t        dj                  |t        |                  )ao  Retrieves a class for managing input for an upload based on file type

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future for the request

        :rtype: class of UploadInputManager
        :returns: The appropriate class to use for managing a specific type of
            input for uploads.
        z&Input {} of type: {} is not supported.)
ro   r   r   ru   rw   r/   rQ   RuntimeErrorformattype)r   rX   upload_manager_resolver_chainr/   upload_manager_clss        r   _get_upload_input_manager_clsz2UploadSubmissionTask._get_upload_input_manager_cls  sx     '&))
% "&&0088"? 	*!//8))	* 4;;g
 	
r   Nc                     | j                  |      || j                  |      }|j                  j                  |j	                  |       |j                  ||      s| j                  ||||||       y| j                  ||||||       y)a  
        :param client: The client associated with the transfer manager

        :type config: s3transfer.manager.TransferConfig
        :param config: The transfer config associated with the transfer
            manager

        :type osutil: s3transfer.utils.OSUtil
        :param osutil: The os utility associated to the transfer manager

        :type request_executor: s3transfer.futures.BoundedExecutor
        :param request_executor: The request executor associated with the
            transfer manager

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future associated with the
            transfer request that tasks are being submitted for
        N)r   r.   ru   ry   rY   r]   _submit_upload_request_submit_multipart_request)r   clientr\   rI   request_executorrX   rJ   upload_input_managers           r   _submitzUploadSubmissionTask._submit.  s    6 
tAA 

$,,.? A
 $$, 66G $==V
 '' $ ** $r   c                 "   |j                   j                  }| j                  |d      }| j                  j	                  |t        | j                  ||j                  |      |j                  |j                  |j                  dd      |       y )Nr   )r   r/   bucketkey
extra_argsT)r0   main_kwargsis_finaltag)
ru   rw   _get_upload_task_tagr.   submitPutObjectTaskr_   r   r   r   )	r   r   r\   rI   r   rX   r   rw   put_object_tags	            r   r   z+UploadSubmissionTask._submit_upload_requestg  s     $((22	 22 ,

 	""))%)%?%?$3GG'  (..$=="+"6"6   	* 	
r   c                 z   |j                   j                  }| j                  j                  |t	        | j                  ||j
                  |j                  |j                  d            }g }	| j                  |j                        }
| j                  |d      }|j                   j                  }t               }|j                  |j                  |      }|j                  ||      }|D ]e  \  }}|	j                  | j                  j                  |t!        | j                  |||j
                  |j                  ||
dd|i      |             g | j#                  |j                        }| j                  j                  |t%        | j                  ||j
                  |j                  |d||	dd	
             y )N)r   r   r   r   )r0   r   upload_part)r   r/   r   r   r   r   	upload_id)r0   r   pending_main_kwargsr   )r   partsT)r0   r   r   r   )ru   rw   r.   r   r	   r   r   r   _extra_upload_part_argsr   ry   r   adjust_chunksizemultipart_chunksizerb   appendUploadPartTask_extra_complete_multipart_argsr   )r   r   r\   rI   r   rX   r   rw   create_multipart_futurepart_futuresextra_part_argsupload_part_tagry   adjusterra   part_iteratorr   r/   complete_multipart_extra_argss                      r   r   z.UploadSubmissionTask._submit_multipart_request  s    $((22	 #'"<"<"C"C%%)%?%?$'..$=="+"6"6	#
 66y7K7KL 33 -
 ##(($&--f.H.H$O	,EEY
 %2 	 K**11$"-1-G-G&,'.&/&6&6#,==+6*9% ()@- (! 2 	, )-(K(K  )
% 	""))'%)%?%?$'..$=="?	 "9)% 	
r   c                 .    t        || j                        S r   )r   UPLOAD_PART_ARGSr   r   s     r   r   z,UploadSubmissionTask._extra_upload_part_args  s     !T-B-BCCr   c                 .    t        || j                        S r   )r   COMPLETE_MULTIPART_ARGSr   s     r   r   z3UploadSubmissionTask._extra_complete_multipart_args  s     T-I-IJJr   c                 8    d }|j                  |      rt        }|S r   )rU   r   )r   r   rT   r   s       r   r   z)UploadSubmissionTask._get_upload_task_tag  s    55nE&C
r   r   )r&   r'   r(   rC   r   r   r   r   r   r   r   r   r   r)   r   r   r   r      sD    8
B 7r!
FT
lD
Kr   r   c                       e Zd ZdZd Zy)r   z Task to do a nonmultipart uploadc                 ^    |5 } |j                   d|||d| ddd       y# 1 sw Y   yxY w)aP  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.
        )BucketKeyBodyNr)   )r   )r   r   r/   r   r   r   r   s          r   _mainzPutObjectTask._main  s>      	OFNV4N:N	O 	O 	Os   #,Nr&   r'   r(   rC   r   r)   r   r   r   r     s    *
Or   r   c                       e Zd ZdZd Zy)r   z+Task to upload a part in a multipart uploadc           
          |5 } |j                   d|||||d|}	ddd       	d   }
|
|d}d|v r$|d   j                         }d| }||	v r|	|   ||<   |S # 1 sw Y   =xY w)a  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param upload_id: The id of the upload
        :param part_number: The number representing the part of the multipart
            upload
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.

        :rtype: dict
        :returns: A dictionary representing a part::

            {'Etag': etag_value, 'PartNumber': part_number}

            This value can be appended to a list to be used to complete
            the multipart upload.
        )r   r   UploadId
PartNumberr   NETag)r  r  r   Checksumr)   )r   upper)r   r   r/   r   r   r   r   r   r   responseetagpart_metadataalgorithm_namechecksum_members                 r   r   zUploadPartTask._main  s    ,  	)v)) "& H	 !%[A*,'(;<BBDN ((89O(*19/1Jo.!	 	s   AA!Nr   r)   r   r   r   r     s
    5&r   r   )r   ior   s3transfer.compatr   r   s3transfer.futuresr   s3transfer.tasksr   r	   r
   r   s3transfer.utilsr   r   r   r   r   r+   rE   ro   r   r   r   r   r   r)   r   r   <module>r     s      0 3   B) )XxN xNvZL!3 ZLz.!; .bG
$6 G
Tl> l^OD O )T )r   